2024-11-21 11:13:33,275 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-21 11:13:33,290 main DEBUG Took 0.012958 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-21 11:13:33,291 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-21 11:13:33,291 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-21 11:13:33,292 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-21 11:13:33,294 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 11:13:33,303 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-21 11:13:33,318 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 11:13:33,320 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 11:13:33,321 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 11:13:33,321 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 11:13:33,322 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 11:13:33,322 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 11:13:33,323 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 11:13:33,324 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 11:13:33,324 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 11:13:33,325 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 11:13:33,326 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 11:13:33,326 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 11:13:33,327 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 11:13:33,327 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 11:13:33,328 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 11:13:33,328 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 11:13:33,329 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 11:13:33,329 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 11:13:33,330 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 11:13:33,331 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 11:13:33,331 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 11:13:33,332 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 11:13:33,332 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 11:13:33,333 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 11:13:33,333 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 11:13:33,334 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-21 11:13:33,336 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 11:13:33,337 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-21 11:13:33,340 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-21 11:13:33,340 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-21 11:13:33,342 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-21 11:13:33,342 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-21 11:13:33,353 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-21 11:13:33,357 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-21 11:13:33,359 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-21 11:13:33,360 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-21 11:13:33,360 main DEBUG createAppenders(={Console}) 2024-11-21 11:13:33,361 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-21 11:13:33,361 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-21 11:13:33,362 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-21 11:13:33,363 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-21 11:13:33,363 main DEBUG OutputStream closed 2024-11-21 11:13:33,363 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-21 11:13:33,364 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-21 11:13:33,364 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-21 11:13:33,443 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-21 11:13:33,445 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-21 11:13:33,446 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-21 11:13:33,447 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-21 11:13:33,448 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-21 11:13:33,448 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-21 11:13:33,448 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-21 11:13:33,449 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-21 11:13:33,449 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-21 11:13:33,449 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-21 11:13:33,449 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-21 11:13:33,450 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-21 11:13:33,450 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-21 11:13:33,450 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-21 11:13:33,451 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-21 11:13:33,451 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-21 11:13:33,451 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-21 11:13:33,452 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-21 11:13:33,454 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21 11:13:33,455 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-21 11:13:33,455 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-21 11:13:33,456 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-21T11:13:33,687 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c 2024-11-21 11:13:33,690 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-21 11:13:33,690 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21T11:13:33,698 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-21T11:13:33,717 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-21T11:13:33,731 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c/cluster_eb393522-c18a-83a9-045f-e99da161695d, deleteOnExit=true 2024-11-21T11:13:33,732 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-21T11:13:33,733 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c/test.cache.data in system properties and HBase conf 2024-11-21T11:13:33,733 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c/hadoop.tmp.dir in system properties and HBase conf 2024-11-21T11:13:33,734 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c/hadoop.log.dir in system properties and HBase conf 2024-11-21T11:13:33,734 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-21T11:13:33,735 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-21T11:13:33,735 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-21T11:13:33,823 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-21T11:13:33,910 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T11:13:33,915 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-21T11:13:33,915 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-21T11:13:33,916 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-21T11:13:33,917 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-21T11:13:33,917 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-21T11:13:33,918 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-21T11:13:33,919 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-21T11:13:33,919 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-21T11:13:33,920 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-21T11:13:33,920 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c/nfs.dump.dir in system properties and HBase conf 2024-11-21T11:13:33,921 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c/java.io.tmpdir in system properties and HBase conf 2024-11-21T11:13:33,921 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-21T11:13:33,922 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-21T11:13:33,923 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-21T11:13:34,861 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-21T11:13:34,942 INFO [Time-limited test {}] log.Log(170): Logging initialized @2388ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-21T11:13:35,020 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T11:13:35,088 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T11:13:35,109 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T11:13:35,110 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T11:13:35,111 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-21T11:13:35,125 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T11:13:35,127 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c/hadoop.log.dir/,AVAILABLE} 2024-11-21T11:13:35,129 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T11:13:35,353 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c/java.io.tmpdir/jetty-localhost-46479-hadoop-hdfs-3_4_1-tests_jar-_-any-13382298111328113510/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-21T11:13:35,366 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:46479} 2024-11-21T11:13:35,366 INFO [Time-limited test {}] server.Server(415): Started @2813ms 2024-11-21T11:13:35,767 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T11:13:35,774 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T11:13:35,775 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T11:13:35,775 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T11:13:35,775 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-21T11:13:35,776 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c/hadoop.log.dir/,AVAILABLE} 2024-11-21T11:13:35,777 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T11:13:35,909 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c/java.io.tmpdir/jetty-localhost-40585-hadoop-hdfs-3_4_1-tests_jar-_-any-952145390950751680/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T11:13:35,910 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:40585} 2024-11-21T11:13:35,910 INFO [Time-limited test {}] server.Server(415): Started @3357ms 2024-11-21T11:13:35,967 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-21T11:13:36,478 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c/cluster_eb393522-c18a-83a9-045f-e99da161695d/data/data2/current/BP-1215574063-172.17.0.2-1732187614556/current, will proceed with Du for space computation calculation, 2024-11-21T11:13:36,478 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c/cluster_eb393522-c18a-83a9-045f-e99da161695d/data/data1/current/BP-1215574063-172.17.0.2-1732187614556/current, will proceed with Du for space computation calculation, 2024-11-21T11:13:36,517 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-21T11:13:36,576 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfd598f204c356453 with lease ID 0x1447d2c594751e17: Processing first storage report for DS-13ddce4f-a5e3-4921-b1a1-430d40592b09 from datanode DatanodeRegistration(127.0.0.1:45789, datanodeUuid=f035d393-e900-4f4f-94a0-7e3e72eeadd3, infoPort=39247, infoSecurePort=0, ipcPort=44751, storageInfo=lv=-57;cid=testClusterID;nsid=1801393481;c=1732187614556) 2024-11-21T11:13:36,578 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfd598f204c356453 with lease ID 0x1447d2c594751e17: from storage DS-13ddce4f-a5e3-4921-b1a1-430d40592b09 node DatanodeRegistration(127.0.0.1:45789, datanodeUuid=f035d393-e900-4f4f-94a0-7e3e72eeadd3, infoPort=39247, infoSecurePort=0, ipcPort=44751, storageInfo=lv=-57;cid=testClusterID;nsid=1801393481;c=1732187614556), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-21T11:13:36,578 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfd598f204c356453 with lease ID 0x1447d2c594751e17: Processing first storage report for DS-93419977-21f4-44c8-a8d3-83f5d71b9d1e from datanode DatanodeRegistration(127.0.0.1:45789, datanodeUuid=f035d393-e900-4f4f-94a0-7e3e72eeadd3, infoPort=39247, infoSecurePort=0, ipcPort=44751, storageInfo=lv=-57;cid=testClusterID;nsid=1801393481;c=1732187614556) 2024-11-21T11:13:36,579 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfd598f204c356453 with lease ID 0x1447d2c594751e17: from storage DS-93419977-21f4-44c8-a8d3-83f5d71b9d1e node DatanodeRegistration(127.0.0.1:45789, datanodeUuid=f035d393-e900-4f4f-94a0-7e3e72eeadd3, infoPort=39247, infoSecurePort=0, ipcPort=44751, storageInfo=lv=-57;cid=testClusterID;nsid=1801393481;c=1732187614556), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-21T11:13:36,660 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c 2024-11-21T11:13:36,743 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c/cluster_eb393522-c18a-83a9-045f-e99da161695d/zookeeper_0, clientPort=51368, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c/cluster_eb393522-c18a-83a9-045f-e99da161695d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c/cluster_eb393522-c18a-83a9-045f-e99da161695d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-21T11:13:36,754 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51368 2024-11-21T11:13:36,766 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T11:13:36,769 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T11:13:36,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741825_1001 (size=7) 2024-11-21T11:13:37,405 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322 with version=8 2024-11-21T11:13:37,405 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/hbase-staging 2024-11-21T11:13:37,507 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-21T11:13:37,762 INFO [Time-limited test {}] client.ConnectionUtils(128): master/7b462513bfc2:0 server-side Connection retries=45 2024-11-21T11:13:37,773 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T11:13:37,774 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T11:13:37,779 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T11:13:37,779 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T11:13:37,779 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T11:13:37,945 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-21T11:13:38,007 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-21T11:13:38,017 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-21T11:13:38,021 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T11:13:38,049 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3070 (auto-detected) 2024-11-21T11:13:38,051 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-21T11:13:38,070 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36543 2024-11-21T11:13:38,093 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36543 connecting to ZooKeeper ensemble=127.0.0.1:51368 2024-11-21T11:13:38,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:365430x0, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T11:13:38,130 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36543-0x1013a3b60600000 connected 2024-11-21T11:13:38,162 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T11:13:38,164 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T11:13:38,173 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36543-0x1013a3b60600000, quorum=127.0.0.1:51368, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T11:13:38,177 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322, hbase.cluster.distributed=false 2024-11-21T11:13:38,199 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36543-0x1013a3b60600000, quorum=127.0.0.1:51368, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T11:13:38,204 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36543 2024-11-21T11:13:38,205 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36543 2024-11-21T11:13:38,205 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36543 2024-11-21T11:13:38,206 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36543 2024-11-21T11:13:38,206 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36543 2024-11-21T11:13:38,319 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7b462513bfc2:0 server-side Connection retries=45 2024-11-21T11:13:38,321 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T11:13:38,322 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T11:13:38,322 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T11:13:38,322 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T11:13:38,322 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T11:13:38,325 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T11:13:38,328 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T11:13:38,329 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33417 2024-11-21T11:13:38,334 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33417 connecting to ZooKeeper ensemble=127.0.0.1:51368 2024-11-21T11:13:38,336 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T11:13:38,340 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T11:13:38,350 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:334170x0, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T11:13:38,351 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:334170x0, quorum=127.0.0.1:51368, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T11:13:38,352 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33417-0x1013a3b60600001 connected 2024-11-21T11:13:38,358 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T11:13:38,375 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T11:13:38,379 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33417-0x1013a3b60600001, quorum=127.0.0.1:51368, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T11:13:38,385 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33417-0x1013a3b60600001, quorum=127.0.0.1:51368, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T11:13:38,386 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33417 2024-11-21T11:13:38,387 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33417 2024-11-21T11:13:38,388 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33417 2024-11-21T11:13:38,389 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33417 2024-11-21T11:13:38,393 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33417 2024-11-21T11:13:38,408 DEBUG [M:0;7b462513bfc2:36543 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;7b462513bfc2:36543 2024-11-21T11:13:38,409 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/7b462513bfc2,36543,1732187617563 2024-11-21T11:13:38,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33417-0x1013a3b60600001, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T11:13:38,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36543-0x1013a3b60600000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T11:13:38,420 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36543-0x1013a3b60600000, quorum=127.0.0.1:51368, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/7b462513bfc2,36543,1732187617563 2024-11-21T11:13:38,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33417-0x1013a3b60600001, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T11:13:38,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36543-0x1013a3b60600000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:13:38,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33417-0x1013a3b60600001, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:13:38,447 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36543-0x1013a3b60600000, quorum=127.0.0.1:51368, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-21T11:13:38,449 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/7b462513bfc2,36543,1732187617563 from backup master directory 2024-11-21T11:13:38,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36543-0x1013a3b60600000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/7b462513bfc2,36543,1732187617563 2024-11-21T11:13:38,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33417-0x1013a3b60600001, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T11:13:38,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36543-0x1013a3b60600000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T11:13:38,455 WARN [master/7b462513bfc2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T11:13:38,455 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=7b462513bfc2,36543,1732187617563 2024-11-21T11:13:38,458 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-21T11:13:38,460 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-21T11:13:38,527 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/hbase.id] with ID: 639acd60-369b-4575-8578-7b7e510d276f 2024-11-21T11:13:38,527 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/.tmp/hbase.id 2024-11-21T11:13:38,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741826_1002 (size=42) 2024-11-21T11:13:38,946 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/.tmp/hbase.id]:[hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/hbase.id] 2024-11-21T11:13:39,000 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T11:13:39,005 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-21T11:13:39,026 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-21T11:13:39,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36543-0x1013a3b60600000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:13:39,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33417-0x1013a3b60600001, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:13:39,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741827_1003 (size=196) 2024-11-21T11:13:39,067 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T11:13:39,070 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-21T11:13:39,087 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:39,091 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T11:13:39,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741828_1004 (size=1189) 2024-11-21T11:13:39,141 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/MasterData/data/master/store 2024-11-21T11:13:39,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741829_1005 (size=34) 2024-11-21T11:13:39,169 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-21T11:13:39,173 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:13:39,174 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T11:13:39,174 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T11:13:39,174 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T11:13:39,176 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T11:13:39,176 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T11:13:39,177 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T11:13:39,178 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732187619174Disabling compacts and flushes for region at 1732187619174Disabling writes for close at 1732187619176 (+2 ms)Writing region close event to WAL at 1732187619177 (+1 ms)Closed at 1732187619177 2024-11-21T11:13:39,181 WARN [master/7b462513bfc2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/MasterData/data/master/store/.initializing 2024-11-21T11:13:39,181 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/MasterData/WALs/7b462513bfc2,36543,1732187617563 2024-11-21T11:13:39,190 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T11:13:39,205 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7b462513bfc2%2C36543%2C1732187617563, suffix=, logDir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/MasterData/WALs/7b462513bfc2,36543,1732187617563, archiveDir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/MasterData/oldWALs, maxLogs=10 2024-11-21T11:13:39,229 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/MasterData/WALs/7b462513bfc2,36543,1732187617563/7b462513bfc2%2C36543%2C1732187617563.1732187619210, exclude list is [], retry=0 2024-11-21T11:13:39,246 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45789,DS-13ddce4f-a5e3-4921-b1a1-430d40592b09,DISK] 2024-11-21T11:13:39,249 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-21T11:13:39,286 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/MasterData/WALs/7b462513bfc2,36543,1732187617563/7b462513bfc2%2C36543%2C1732187617563.1732187619210 2024-11-21T11:13:39,287 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39247:39247)] 2024-11-21T11:13:39,288 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-21T11:13:39,289 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:13:39,292 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T11:13:39,293 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T11:13:39,330 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T11:13:39,359 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-21T11:13:39,363 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:13:39,366 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T11:13:39,367 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T11:13:39,370 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-21T11:13:39,371 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:13:39,372 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:13:39,372 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T11:13:39,374 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-21T11:13:39,375 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:13:39,376 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:13:39,376 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T11:13:39,378 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-21T11:13:39,378 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:13:39,379 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:13:39,380 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T11:13:39,383 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T11:13:39,384 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T11:13:39,389 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T11:13:39,390 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T11:13:39,393 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T11:13:39,396 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T11:13:39,400 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T11:13:39,402 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73158694, jitterRate=0.09014949202537537}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T11:13:39,407 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732187619305Initializing all the Stores at 1732187619308 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732187619308Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187619309 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187619309Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187619309Cleaning up temporary data from old regions at 1732187619390 (+81 ms)Region opened successfully at 1732187619407 (+17 ms) 2024-11-21T11:13:39,408 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-21T11:13:39,446 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@570cae89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7b462513bfc2/172.17.0.2:0 2024-11-21T11:13:39,483 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-21T11:13:39,495 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-21T11:13:39,495 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-21T11:13:39,498 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T11:13:39,499 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-21T11:13:39,504 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-21T11:13:39,504 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-21T11:13:39,531 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-21T11:13:39,540 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36543-0x1013a3b60600000, quorum=127.0.0.1:51368, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-21T11:13:39,542 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-21T11:13:39,544 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-21T11:13:39,546 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36543-0x1013a3b60600000, quorum=127.0.0.1:51368, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-21T11:13:39,547 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-21T11:13:39,550 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-21T11:13:39,554 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36543-0x1013a3b60600000, quorum=127.0.0.1:51368, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-21T11:13:39,556 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-21T11:13:39,558 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36543-0x1013a3b60600000, quorum=127.0.0.1:51368, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-21T11:13:39,559 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-21T11:13:39,577 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36543-0x1013a3b60600000, quorum=127.0.0.1:51368, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-21T11:13:39,579 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-21T11:13:39,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33417-0x1013a3b60600001, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T11:13:39,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36543-0x1013a3b60600000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T11:13:39,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33417-0x1013a3b60600001, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:13:39,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36543-0x1013a3b60600000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:13:39,585 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=7b462513bfc2,36543,1732187617563, sessionid=0x1013a3b60600000, setting cluster-up flag (Was=false) 2024-11-21T11:13:39,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33417-0x1013a3b60600001, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:13:39,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36543-0x1013a3b60600000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:13:39,603 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-21T11:13:39,605 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7b462513bfc2,36543,1732187617563 2024-11-21T11:13:39,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33417-0x1013a3b60600001, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:13:39,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36543-0x1013a3b60600000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:13:39,621 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-21T11:13:39,623 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7b462513bfc2,36543,1732187617563 2024-11-21T11:13:39,629 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-21T11:13:39,698 INFO [RS:0;7b462513bfc2:33417 {}] regionserver.HRegionServer(746): ClusterId : 639acd60-369b-4575-8578-7b7e510d276f 2024-11-21T11:13:39,701 DEBUG [RS:0;7b462513bfc2:33417 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T11:13:39,706 DEBUG [RS:0;7b462513bfc2:33417 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T11:13:39,705 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T11:13:39,706 DEBUG [RS:0;7b462513bfc2:33417 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T11:13:39,709 DEBUG [RS:0;7b462513bfc2:33417 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T11:13:39,709 DEBUG [RS:0;7b462513bfc2:33417 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3936a107, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7b462513bfc2/172.17.0.2:0 2024-11-21T11:13:39,715 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-21T11:13:39,722 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-21T11:13:39,724 DEBUG [RS:0;7b462513bfc2:33417 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;7b462513bfc2:33417 2024-11-21T11:13:39,727 INFO [RS:0;7b462513bfc2:33417 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T11:13:39,727 INFO [RS:0;7b462513bfc2:33417 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T11:13:39,727 DEBUG [RS:0;7b462513bfc2:33417 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T11:13:39,730 INFO [RS:0;7b462513bfc2:33417 {}] regionserver.HRegionServer(2659): reportForDuty to master=7b462513bfc2,36543,1732187617563 with port=33417, startcode=1732187618276 2024-11-21T11:13:39,728 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 7b462513bfc2,36543,1732187617563 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-21T11:13:39,735 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/7b462513bfc2:0, corePoolSize=5, maxPoolSize=5 2024-11-21T11:13:39,735 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/7b462513bfc2:0, corePoolSize=5, maxPoolSize=5 2024-11-21T11:13:39,735 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/7b462513bfc2:0, corePoolSize=5, maxPoolSize=5 2024-11-21T11:13:39,735 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/7b462513bfc2:0, corePoolSize=5, maxPoolSize=5 2024-11-21T11:13:39,735 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/7b462513bfc2:0, corePoolSize=10, maxPoolSize=10 2024-11-21T11:13:39,736 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:13:39,736 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/7b462513bfc2:0, corePoolSize=2, maxPoolSize=2 2024-11-21T11:13:39,736 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:13:39,739 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732187649739 2024-11-21T11:13:39,741 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-21T11:13:39,742 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-21T11:13:39,742 DEBUG [RS:0;7b462513bfc2:33417 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T11:13:39,744 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T11:13:39,744 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-21T11:13:39,745 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-21T11:13:39,746 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-21T11:13:39,746 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-21T11:13:39,746 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-21T11:13:39,747 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T11:13:39,750 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-21T11:13:39,751 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-21T11:13:39,751 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-21T11:13:39,752 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:13:39,752 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-21T11:13:39,753 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-21T11:13:39,753 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-21T11:13:39,755 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/7b462513bfc2:0:becomeActiveMaster-HFileCleaner.large.0-1732187619755,5,FailOnTimeoutGroup] 2024-11-21T11:13:39,756 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/7b462513bfc2:0:becomeActiveMaster-HFileCleaner.small.0-1732187619755,5,FailOnTimeoutGroup] 2024-11-21T11:13:39,756 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T11:13:39,756 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-21T11:13:39,758 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-21T11:13:39,758 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-21T11:13:39,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741831_1007 (size=1321) 2024-11-21T11:13:39,777 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-21T11:13:39,777 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322 2024-11-21T11:13:39,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741832_1008 (size=32) 2024-11-21T11:13:39,791 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:13:39,792 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37479, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T11:13:39,793 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-21T11:13:39,796 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T11:13:39,796 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:13:39,797 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T11:13:39,798 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-21T11:13:39,800 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36543 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7b462513bfc2,33417,1732187618276 2024-11-21T11:13:39,800 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T11:13:39,800 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:13:39,801 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T11:13:39,801 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-21T11:13:39,803 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36543 {}] master.ServerManager(517): Registering regionserver=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:39,803 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T11:13:39,803 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:13:39,804 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T11:13:39,804 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-21T11:13:39,807 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T11:13:39,807 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:13:39,808 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T11:13:39,808 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T11:13:39,809 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/hbase/meta/1588230740 2024-11-21T11:13:39,810 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/hbase/meta/1588230740 2024-11-21T11:13:39,813 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T11:13:39,813 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T11:13:39,814 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T11:13:39,816 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T11:13:39,820 DEBUG [RS:0;7b462513bfc2:33417 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322 2024-11-21T11:13:39,820 DEBUG [RS:0;7b462513bfc2:33417 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33769 2024-11-21T11:13:39,820 DEBUG [RS:0;7b462513bfc2:33417 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T11:13:39,820 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T11:13:39,821 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66552385, jitterRate=-0.00829218327999115}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T11:13:39,824 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732187619791Initializing all the Stores at 1732187619793 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732187619793Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732187619793Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187619793Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732187619793Cleaning up temporary data from old regions at 1732187619813 (+20 ms)Region opened successfully at 1732187619824 (+11 ms) 2024-11-21T11:13:39,824 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T11:13:39,824 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T11:13:39,825 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T11:13:39,825 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T11:13:39,825 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T11:13:39,826 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T11:13:39,826 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732187619824Disabling compacts and flushes for region at 1732187619824Disabling writes for close at 1732187619825 (+1 ms)Writing region close event to WAL at 1732187619826 (+1 ms)Closed at 1732187619826 2024-11-21T11:13:39,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36543-0x1013a3b60600000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T11:13:39,828 DEBUG [RS:0;7b462513bfc2:33417 {}] zookeeper.ZKUtil(111): regionserver:33417-0x1013a3b60600001, quorum=127.0.0.1:51368, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7b462513bfc2,33417,1732187618276 2024-11-21T11:13:39,828 WARN [RS:0;7b462513bfc2:33417 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T11:13:39,829 INFO [RS:0;7b462513bfc2:33417 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T11:13:39,829 DEBUG [RS:0;7b462513bfc2:33417 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/WALs/7b462513bfc2,33417,1732187618276 2024-11-21T11:13:39,830 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7b462513bfc2,33417,1732187618276] 2024-11-21T11:13:39,830 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T11:13:39,830 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-21T11:13:39,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-21T11:13:39,850 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T11:13:39,853 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-21T11:13:39,860 INFO [RS:0;7b462513bfc2:33417 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T11:13:39,876 INFO [RS:0;7b462513bfc2:33417 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T11:13:39,884 INFO [RS:0;7b462513bfc2:33417 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T11:13:39,884 INFO [RS:0;7b462513bfc2:33417 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T11:13:39,885 INFO [RS:0;7b462513bfc2:33417 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T11:13:39,894 INFO [RS:0;7b462513bfc2:33417 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T11:13:39,896 INFO [RS:0;7b462513bfc2:33417 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T11:13:39,896 DEBUG [RS:0;7b462513bfc2:33417 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:13:39,896 DEBUG [RS:0;7b462513bfc2:33417 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:13:39,897 DEBUG [RS:0;7b462513bfc2:33417 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:13:39,897 DEBUG [RS:0;7b462513bfc2:33417 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:13:39,897 DEBUG [RS:0;7b462513bfc2:33417 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:13:39,897 DEBUG [RS:0;7b462513bfc2:33417 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0, corePoolSize=2, maxPoolSize=2 2024-11-21T11:13:39,897 DEBUG [RS:0;7b462513bfc2:33417 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:13:39,897 DEBUG [RS:0;7b462513bfc2:33417 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:13:39,898 DEBUG [RS:0;7b462513bfc2:33417 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:13:39,898 DEBUG [RS:0;7b462513bfc2:33417 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:13:39,898 DEBUG [RS:0;7b462513bfc2:33417 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:13:39,898 DEBUG [RS:0;7b462513bfc2:33417 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:13:39,898 DEBUG [RS:0;7b462513bfc2:33417 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7b462513bfc2:0, corePoolSize=3, maxPoolSize=3 2024-11-21T11:13:39,898 DEBUG [RS:0;7b462513bfc2:33417 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0, corePoolSize=3, maxPoolSize=3 2024-11-21T11:13:39,899 INFO [RS:0;7b462513bfc2:33417 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T11:13:39,899 INFO [RS:0;7b462513bfc2:33417 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T11:13:39,900 INFO [RS:0;7b462513bfc2:33417 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T11:13:39,900 INFO [RS:0;7b462513bfc2:33417 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T11:13:39,901 INFO [RS:0;7b462513bfc2:33417 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T11:13:39,901 INFO [RS:0;7b462513bfc2:33417 {}] hbase.ChoreService(168): Chore ScheduledChore name=7b462513bfc2,33417,1732187618276-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T11:13:39,924 INFO [RS:0;7b462513bfc2:33417 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T11:13:39,926 INFO [RS:0;7b462513bfc2:33417 {}] hbase.ChoreService(168): Chore ScheduledChore name=7b462513bfc2,33417,1732187618276-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T11:13:39,927 INFO [RS:0;7b462513bfc2:33417 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T11:13:39,927 INFO [RS:0;7b462513bfc2:33417 {}] regionserver.Replication(171): 7b462513bfc2,33417,1732187618276 started 2024-11-21T11:13:39,947 INFO [RS:0;7b462513bfc2:33417 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T11:13:39,948 INFO [RS:0;7b462513bfc2:33417 {}] regionserver.HRegionServer(1482): Serving as 7b462513bfc2,33417,1732187618276, RpcServer on 7b462513bfc2/172.17.0.2:33417, sessionid=0x1013a3b60600001 2024-11-21T11:13:39,949 DEBUG [RS:0;7b462513bfc2:33417 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T11:13:39,949 DEBUG [RS:0;7b462513bfc2:33417 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7b462513bfc2,33417,1732187618276 2024-11-21T11:13:39,949 DEBUG [RS:0;7b462513bfc2:33417 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7b462513bfc2,33417,1732187618276' 2024-11-21T11:13:39,949 DEBUG [RS:0;7b462513bfc2:33417 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T11:13:39,950 DEBUG [RS:0;7b462513bfc2:33417 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T11:13:39,951 DEBUG [RS:0;7b462513bfc2:33417 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T11:13:39,951 DEBUG [RS:0;7b462513bfc2:33417 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T11:13:39,951 DEBUG [RS:0;7b462513bfc2:33417 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7b462513bfc2,33417,1732187618276 2024-11-21T11:13:39,951 DEBUG [RS:0;7b462513bfc2:33417 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7b462513bfc2,33417,1732187618276' 2024-11-21T11:13:39,951 DEBUG [RS:0;7b462513bfc2:33417 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T11:13:39,952 DEBUG [RS:0;7b462513bfc2:33417 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T11:13:39,953 DEBUG [RS:0;7b462513bfc2:33417 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T11:13:39,953 INFO [RS:0;7b462513bfc2:33417 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T11:13:39,953 INFO [RS:0;7b462513bfc2:33417 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T11:13:40,005 WARN [7b462513bfc2:36543 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T11:13:40,059 INFO [RS:0;7b462513bfc2:33417 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T11:13:40,062 INFO [RS:0;7b462513bfc2:33417 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7b462513bfc2%2C33417%2C1732187618276, suffix=, logDir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/WALs/7b462513bfc2,33417,1732187618276, archiveDir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/oldWALs, maxLogs=32 2024-11-21T11:13:40,079 DEBUG [RS:0;7b462513bfc2:33417 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/WALs/7b462513bfc2,33417,1732187618276/7b462513bfc2%2C33417%2C1732187618276.1732187620065, exclude list is [], retry=0 2024-11-21T11:13:40,084 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45789,DS-13ddce4f-a5e3-4921-b1a1-430d40592b09,DISK] 2024-11-21T11:13:40,087 INFO [RS:0;7b462513bfc2:33417 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/WALs/7b462513bfc2,33417,1732187618276/7b462513bfc2%2C33417%2C1732187618276.1732187620065 2024-11-21T11:13:40,088 DEBUG [RS:0;7b462513bfc2:33417 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39247:39247)] 2024-11-21T11:13:40,257 DEBUG [7b462513bfc2:36543 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-21T11:13:40,269 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:40,275 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b462513bfc2,33417,1732187618276, state=OPENING 2024-11-21T11:13:40,280 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-21T11:13:40,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36543-0x1013a3b60600000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:13:40,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33417-0x1013a3b60600001, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:13:40,283 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:13:40,283 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:13:40,284 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T11:13:40,286 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,33417,1732187618276}] 2024-11-21T11:13:40,461 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T11:13:40,465 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35293, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T11:13:40,478 INFO [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-21T11:13:40,479 INFO [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T11:13:40,479 INFO [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-21T11:13:40,483 INFO [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7b462513bfc2%2C33417%2C1732187618276.meta, suffix=.meta, logDir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/WALs/7b462513bfc2,33417,1732187618276, archiveDir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/oldWALs, maxLogs=32 2024-11-21T11:13:40,507 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/WALs/7b462513bfc2,33417,1732187618276/7b462513bfc2%2C33417%2C1732187618276.meta.1732187620486.meta, exclude list is [], retry=0 2024-11-21T11:13:40,511 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45789,DS-13ddce4f-a5e3-4921-b1a1-430d40592b09,DISK] 2024-11-21T11:13:40,514 INFO [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/WALs/7b462513bfc2,33417,1732187618276/7b462513bfc2%2C33417%2C1732187618276.meta.1732187620486.meta 2024-11-21T11:13:40,514 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39247:39247)] 2024-11-21T11:13:40,515 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-21T11:13:40,517 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-21T11:13:40,520 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-21T11:13:40,524 INFO [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-21T11:13:40,529 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-21T11:13:40,529 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:13:40,530 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-21T11:13:40,530 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-21T11:13:40,533 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-21T11:13:40,534 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T11:13:40,535 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:13:40,535 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T11:13:40,536 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-21T11:13:40,537 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T11:13:40,537 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:13:40,538 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T11:13:40,538 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-21T11:13:40,540 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T11:13:40,540 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:13:40,540 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T11:13:40,541 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-21T11:13:40,542 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T11:13:40,542 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:13:40,543 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T11:13:40,543 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T11:13:40,544 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/hbase/meta/1588230740 2024-11-21T11:13:40,547 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/hbase/meta/1588230740 2024-11-21T11:13:40,549 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T11:13:40,549 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T11:13:40,550 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T11:13:40,552 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T11:13:40,554 INFO [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60339996, jitterRate=-0.1008639931678772}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T11:13:40,554 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-21T11:13:40,555 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732187620530Writing region info on filesystem at 1732187620530Initializing all the Stores at 1732187620532 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732187620532Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732187620533 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187620533Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732187620533Cleaning up temporary data from old regions at 1732187620549 (+16 ms)Running coprocessor post-open hooks at 1732187620554 (+5 ms)Region opened successfully at 1732187620555 (+1 ms) 2024-11-21T11:13:40,563 INFO [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732187620452 2024-11-21T11:13:40,574 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-21T11:13:40,574 INFO [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-21T11:13:40,576 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:40,578 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b462513bfc2,33417,1732187618276, state=OPEN 2024-11-21T11:13:40,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36543-0x1013a3b60600000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:13:40,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33417-0x1013a3b60600001, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:13:40,586 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:13:40,586 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:13:40,586 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:40,591 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-21T11:13:40,591 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,33417,1732187618276 in 301 msec 2024-11-21T11:13:40,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-21T11:13:40,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 755 msec 2024-11-21T11:13:40,599 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T11:13:40,599 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-21T11:13:40,619 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:13:40,621 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:13:40,642 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:13:40,645 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47695, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:13:40,663 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0020 sec 2024-11-21T11:13:40,664 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732187620664, completionTime=-1 2024-11-21T11:13:40,666 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-21T11:13:40,667 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T11:13:40,693 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-21T11:13:40,693 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732187680693 2024-11-21T11:13:40,693 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732187740693 2024-11-21T11:13:40,694 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-11-21T11:13:40,696 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7b462513bfc2,36543,1732187617563-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T11:13:40,696 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7b462513bfc2,36543,1732187617563-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T11:13:40,696 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7b462513bfc2,36543,1732187617563-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T11:13:40,698 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-7b462513bfc2:36543, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T11:13:40,698 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-21T11:13:40,699 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-21T11:13:40,704 DEBUG [master/7b462513bfc2:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-21T11:13:40,725 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.270sec 2024-11-21T11:13:40,726 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-21T11:13:40,727 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-21T11:13:40,728 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-21T11:13:40,729 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-21T11:13:40,729 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-21T11:13:40,730 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7b462513bfc2,36543,1732187617563-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T11:13:40,730 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7b462513bfc2,36543,1732187617563-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-21T11:13:40,739 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-21T11:13:40,740 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-21T11:13:40,740 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7b462513bfc2,36543,1732187617563-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T11:13:40,808 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:13:40,810 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-21T11:13:40,810 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-21T11:13:40,813 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:13:40,816 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:13:40,824 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:13:40,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:13:40,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:13:40,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36ed2532, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:13:40,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:13:40,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:13:40,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:13:40,835 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:13:40,839 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:13:40,839 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:13:40,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:13:40,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:13:40,852 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:13:40,876 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=7b462513bfc2,36543,1732187617563 2024-11-21T11:13:40,889 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=318, ProcessCount=11, AvailableMemoryMB=5877 2024-11-21T11:13:40,900 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T11:13:40,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 7b462513bfc2,36543,1732187617563 2024-11-21T11:13:40,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4a51e70c 2024-11-21T11:13:40,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T11:13:40,916 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41258, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T11:13:40,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T11:13:40,925 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T11:13:40,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T11:13:40,939 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T11:13:40,942 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-21T11:13:40,943 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:13:40,946 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T11:13:40,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T11:13:40,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741835_1011 (size=1061) 2024-11-21T11:13:41,004 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322 2024-11-21T11:13:41,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741836_1012 (size=53) 2024-11-21T11:13:41,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T11:13:41,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T11:13:41,429 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:13:41,429 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing cb944dfe54fbd2ebb234cb2046a89860, disabling compactions & flushes 2024-11-21T11:13:41,429 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:41,429 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:41,430 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. after waiting 0 ms 2024-11-21T11:13:41,430 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:41,430 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:41,430 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for cb944dfe54fbd2ebb234cb2046a89860: Waiting for close lock at 1732187621429Disabling compacts and flushes for region at 1732187621429Disabling writes for close at 1732187621430 (+1 ms)Writing region close event to WAL at 1732187621430Closed at 1732187621430 2024-11-21T11:13:41,433 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T11:13:41,440 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732187621434"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732187621434"}]},"ts":"1732187621434"} 2024-11-21T11:13:41,448 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T11:13:41,454 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T11:13:41,460 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732187621454"}]},"ts":"1732187621454"} 2024-11-21T11:13:41,470 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T11:13:41,473 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb944dfe54fbd2ebb234cb2046a89860, ASSIGN}] 2024-11-21T11:13:41,476 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb944dfe54fbd2ebb234cb2046a89860, ASSIGN 2024-11-21T11:13:41,479 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb944dfe54fbd2ebb234cb2046a89860, ASSIGN; state=OFFLINE, location=7b462513bfc2,33417,1732187618276; forceNewPlan=false, retain=false 2024-11-21T11:13:41,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T11:13:41,632 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=cb944dfe54fbd2ebb234cb2046a89860, regionState=OPENING, regionLocation=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:41,638 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb944dfe54fbd2ebb234cb2046a89860, ASSIGN because future has completed 2024-11-21T11:13:41,639 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276}] 2024-11-21T11:13:41,833 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:41,833 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} 2024-11-21T11:13:41,834 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:13:41,834 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:13:41,834 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:13:41,834 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:13:41,842 INFO [StoreOpener-cb944dfe54fbd2ebb234cb2046a89860-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:13:41,857 INFO [StoreOpener-cb944dfe54fbd2ebb234cb2046a89860-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T11:13:41,858 INFO [StoreOpener-cb944dfe54fbd2ebb234cb2046a89860-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb944dfe54fbd2ebb234cb2046a89860 columnFamilyName A 2024-11-21T11:13:41,858 DEBUG [StoreOpener-cb944dfe54fbd2ebb234cb2046a89860-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:13:41,860 INFO [StoreOpener-cb944dfe54fbd2ebb234cb2046a89860-1 {}] regionserver.HStore(327): Store=cb944dfe54fbd2ebb234cb2046a89860/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:13:41,860 INFO [StoreOpener-cb944dfe54fbd2ebb234cb2046a89860-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:13:41,866 INFO [StoreOpener-cb944dfe54fbd2ebb234cb2046a89860-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T11:13:41,867 INFO [StoreOpener-cb944dfe54fbd2ebb234cb2046a89860-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb944dfe54fbd2ebb234cb2046a89860 columnFamilyName B 2024-11-21T11:13:41,867 DEBUG [StoreOpener-cb944dfe54fbd2ebb234cb2046a89860-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:13:41,868 INFO [StoreOpener-cb944dfe54fbd2ebb234cb2046a89860-1 {}] regionserver.HStore(327): Store=cb944dfe54fbd2ebb234cb2046a89860/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:13:41,868 INFO [StoreOpener-cb944dfe54fbd2ebb234cb2046a89860-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:13:41,871 INFO [StoreOpener-cb944dfe54fbd2ebb234cb2046a89860-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T11:13:41,872 INFO [StoreOpener-cb944dfe54fbd2ebb234cb2046a89860-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb944dfe54fbd2ebb234cb2046a89860 columnFamilyName C 2024-11-21T11:13:41,877 DEBUG [StoreOpener-cb944dfe54fbd2ebb234cb2046a89860-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:13:41,880 INFO [StoreOpener-cb944dfe54fbd2ebb234cb2046a89860-1 {}] regionserver.HStore(327): Store=cb944dfe54fbd2ebb234cb2046a89860/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:13:41,881 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:41,881 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:13:41,883 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:13:41,884 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:13:41,889 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:13:41,889 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:13:41,890 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T11:13:41,896 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:13:41,902 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T11:13:41,903 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened cb944dfe54fbd2ebb234cb2046a89860; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59959140, jitterRate=-0.10653918981552124}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T11:13:41,903 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:13:41,905 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for cb944dfe54fbd2ebb234cb2046a89860: Running coprocessor pre-open hook at 1732187621835Writing region info on filesystem at 1732187621835Initializing all the Stores at 1732187621837 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187621837Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187621838 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187621838Cleaning up temporary data from old regions at 1732187621889 (+51 ms)Running coprocessor post-open hooks at 1732187621903 (+14 ms)Region opened successfully at 1732187621905 (+2 ms) 2024-11-21T11:13:41,907 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., pid=6, masterSystemTime=1732187621800 2024-11-21T11:13:41,916 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:41,916 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:41,921 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=cb944dfe54fbd2ebb234cb2046a89860, regionState=OPEN, openSeqNum=2, regionLocation=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:41,927 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 because future has completed 2024-11-21T11:13:41,949 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-21T11:13:41,951 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 in 302 msec 2024-11-21T11:13:41,954 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-21T11:13:41,954 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb944dfe54fbd2ebb234cb2046a89860, ASSIGN in 476 msec 2024-11-21T11:13:41,956 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T11:13:41,956 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732187621956"}]},"ts":"1732187621956"} 2024-11-21T11:13:41,961 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T11:13:41,963 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T11:13:41,967 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.0350 sec 2024-11-21T11:13:42,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T11:13:42,119 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:13:42,129 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:13:42,129 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:13:42,129 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:13:42,131 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:13:42,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:13:42,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:13:42,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a220135, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:13:42,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:13:42,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:13:42,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:13:42,139 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:13:42,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@322a9c3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:13:42,142 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:13:42,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:13:42,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:13:42,148 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:13:42,151 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T11:13:42,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 7b462513bfc2,36543,1732187617563 2024-11-21T11:13:42,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@63481454 2024-11-21T11:13:42,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T11:13:42,162 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T11:13:42,169 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:13:42,169 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:13:42,169 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:13:42,171 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:13:42,171 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:13:42,172 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:13:42,172 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b130fb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:13:42,172 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:13:42,172 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:13:42,173 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:13:42,174 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:13:42,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d1d7645, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:13:42,177 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:13:42,177 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:13:42,177 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:13:42,181 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:13:42,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:13:42,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:13:42,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27ff6f12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:13:42,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:13:42,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:13:42,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:13:42,186 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:13:42,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10bc6ecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:13:42,189 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:13:42,189 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:13:42,189 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:13:42,191 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:13:42,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:13:42,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:13:42,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff1f1f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:13:42,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:13:42,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:13:42,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:13:42,197 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41354, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:13:42,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14ba4844, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:13:42,200 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:13:42,200 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:13:42,200 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:13:42,202 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:13:42,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:13:42,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:13:42,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55ae7626, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:13:42,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:13:42,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:13:42,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:13:42,206 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:13:42,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d73963e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:13:42,212 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:13:42,212 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:13:42,212 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:13:42,213 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:13:42,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:13:42,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:13:42,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b6288b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:13:42,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:13:42,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:13:42,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:13:42,217 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:13:42,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@703aa885, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:13:42,225 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:13:42,225 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:13:42,225 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:13:42,226 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:13:42,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:13:42,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:13:42,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f820931, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:13:42,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:13:42,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:13:42,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:13:42,230 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:13:42,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8f5797d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:13:42,232 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:13:42,232 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:13:42,232 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:13:42,233 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:13:42,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:13:42,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:13:42,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@788043de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:13:42,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:13:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:13:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:13:42,237 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:13:42,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@421973e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:13:42,241 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:13:42,241 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:13:42,241 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:13:42,242 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:13:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:13:42,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:13:42,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78238dbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:13:42,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:13:42,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:13:42,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:13:42,245 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:13:42,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76c8af47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:13:42,248 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:13:42,249 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:13:42,249 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:13:42,250 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:13:42,250 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:13:42,250 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:13:42,251 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13c29b6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:13:42,251 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:13:42,251 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:13:42,251 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:13:42,253 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:13:42,254 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78aaa9c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:13:42,260 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:13:42,260 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:13:42,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:13:42,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:13:42,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:13:42,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:13:42,264 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40396, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:13:42,264 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:13:42,265 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:13:42,266 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:13:42,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:13:42,268 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:13:42,269 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:13:42,269 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:13:42,270 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:13:42,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:13:42,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:13:42,272 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:13:42,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:13:42,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:13:42,273 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:13:42,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:13:42,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2] 2024-11-21T11:13:42,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2] 2024-11-21T11:13:42,274 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40434, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:13:42,275 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:13:42,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2] 2024-11-21T11:13:42,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2] 2024-11-21T11:13:42,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2] 2024-11-21T11:13:42,280 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:13:42,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2] 2024-11-21T11:13:42,286 DEBUG [Thread-151 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:13:42,286 DEBUG [Thread-151 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:13:42,288 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:13:42,289 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:13:42,289 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:13:42,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:13:42,291 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:13:42,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2] 2024-11-21T11:13:42,291 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:13:42,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:13:42,294 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40458, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:13:42,294 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:13:42,300 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:13:42,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2] 2024-11-21T11:13:42,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2] 2024-11-21T11:13:42,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-21T11:13:42,309 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:13:42,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T11:13:42,313 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:13:42,316 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:13:42,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:13:42,355 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb944dfe54fbd2ebb234cb2046a89860 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T11:13:42,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=A 2024-11-21T11:13:42,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:42,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=B 2024-11-21T11:13:42,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:42,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=C 2024-11-21T11:13:42,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:42,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T11:13:42,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T11:13:42,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:42,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:42,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:42,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:42,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:42,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:42,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:42,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:42,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187682505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:42,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187682504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:42,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:42,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187682513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:42,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:42,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40396 deadline: 1732187682528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:42,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/31ab7a47424249f39fbf9a067fafdf8f is 50, key is test_row_0/A:col10/1732187622345/Put/seqid=0 2024-11-21T11:13:42,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:42,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187682529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:42,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:42,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:42,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:42,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:42,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:42,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:42,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:42,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:42,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:42,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:42,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:42,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:42,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:42,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:42,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:42,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741837_1013 (size=12001) 2024-11-21T11:13:42,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/31ab7a47424249f39fbf9a067fafdf8f 2024-11-21T11:13:42,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T11:13:42,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:42,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187682666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:42,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:42,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:42,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:42,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:42,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187682667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:42,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T11:13:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:42,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:42,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187682668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:42,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:42,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187682672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:42,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:42,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:42,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:42,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:42,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:42,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:42,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40396 deadline: 1732187682668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:42,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:42,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:42,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:42,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:42,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:42,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:42,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:42,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:42,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:42,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/8b6f85bef0074899bcb1d517efe1e818 is 50, key is test_row_0/B:col10/1732187622345/Put/seqid=0 2024-11-21T11:13:42,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741838_1014 (size=12001) 2024-11-21T11:13:42,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/8b6f85bef0074899bcb1d517efe1e818 2024-11-21T11:13:42,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/4f294d546e134601a7125063f955e677 is 50, key is test_row_0/C:col10/1732187622345/Put/seqid=0 2024-11-21T11:13:42,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741839_1015 (size=12001) 2024-11-21T11:13:42,801 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/4f294d546e134601a7125063f955e677 2024-11-21T11:13:42,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/31ab7a47424249f39fbf9a067fafdf8f as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/31ab7a47424249f39fbf9a067fafdf8f 2024-11-21T11:13:42,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T11:13:42,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:42,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:42,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:42,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:42,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:42,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:42,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/31ab7a47424249f39fbf9a067fafdf8f, entries=150, sequenceid=14, filesize=11.7 K 2024-11-21T11:13:42,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/8b6f85bef0074899bcb1d517efe1e818 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/8b6f85bef0074899bcb1d517efe1e818 2024-11-21T11:13:42,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/8b6f85bef0074899bcb1d517efe1e818, entries=150, sequenceid=14, filesize=11.7 K 2024-11-21T11:13:42,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/4f294d546e134601a7125063f955e677 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/4f294d546e134601a7125063f955e677 2024-11-21T11:13:42,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:42,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187682876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:42,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:42,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:42,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:42,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:42,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40396 deadline: 1732187682890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:42,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:42,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:42,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:42,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:42,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187682890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:42,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:42,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:42,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:42,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:42,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187682890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:42,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:42,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:42,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:42,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:42,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187682891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:42,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:42,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:42,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:42,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/4f294d546e134601a7125063f955e677, entries=150, sequenceid=14, filesize=11.7 K 2024-11-21T11:13:42,927 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for cb944dfe54fbd2ebb234cb2046a89860 in 569ms, sequenceid=14, compaction requested=false 2024-11-21T11:13:42,929 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-21T11:13:42,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:42,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T11:13:43,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T11:13:43,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:43,003 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing cb944dfe54fbd2ebb234cb2046a89860 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-21T11:13:43,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=A 2024-11-21T11:13:43,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:43,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=B 2024-11-21T11:13:43,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:43,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=C 2024-11-21T11:13:43,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:43,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/4506b49795554c2e951f90a566b3beb8 is 50, key is test_row_0/A:col10/1732187622518/Put/seqid=0 2024-11-21T11:13:43,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741840_1016 (size=12001) 2024-11-21T11:13:43,091 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/4506b49795554c2e951f90a566b3beb8 2024-11-21T11:13:43,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/fad3b6a8d0b74ab99c4cdf90aa668f83 is 50, key is test_row_0/B:col10/1732187622518/Put/seqid=0 2024-11-21T11:13:43,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741841_1017 (size=12001) 2024-11-21T11:13:43,156 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/fad3b6a8d0b74ab99c4cdf90aa668f83 2024-11-21T11:13:43,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/812a707b47304e0eb57ef128c28ea027 is 50, key is test_row_0/C:col10/1732187622518/Put/seqid=0 2024-11-21T11:13:43,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741842_1018 (size=12001) 2024-11-21T11:13:43,208 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/812a707b47304e0eb57ef128c28ea027 2024-11-21T11:13:43,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:13:43,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:43,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/4506b49795554c2e951f90a566b3beb8 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/4506b49795554c2e951f90a566b3beb8 2024-11-21T11:13:43,251 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/4506b49795554c2e951f90a566b3beb8, entries=150, sequenceid=34, filesize=11.7 K 2024-11-21T11:13:43,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/fad3b6a8d0b74ab99c4cdf90aa668f83 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/fad3b6a8d0b74ab99c4cdf90aa668f83 2024-11-21T11:13:43,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:43,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187683254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:43,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:43,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:43,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187683257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:43,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:43,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:43,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187683261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:43,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:43,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:43,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40396 deadline: 1732187683265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:43,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:43,273 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/fad3b6a8d0b74ab99c4cdf90aa668f83, entries=150, sequenceid=34, filesize=11.7 K 2024-11-21T11:13:43,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:43,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187683267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:43,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:43,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/812a707b47304e0eb57ef128c28ea027 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/812a707b47304e0eb57ef128c28ea027 2024-11-21T11:13:43,292 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/812a707b47304e0eb57ef128c28ea027, entries=150, sequenceid=34, filesize=11.7 K 2024-11-21T11:13:43,297 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for cb944dfe54fbd2ebb234cb2046a89860 in 293ms, sequenceid=34, compaction requested=false 2024-11-21T11:13:43,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:43,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:43,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-21T11:13:43,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-21T11:13:43,304 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-21T11:13:43,305 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 985 msec 2024-11-21T11:13:43,313 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.0060 sec 2024-11-21T11:13:43,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:13:43,392 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb944dfe54fbd2ebb234cb2046a89860 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-21T11:13:43,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=A 2024-11-21T11:13:43,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:43,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=B 2024-11-21T11:13:43,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:43,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=C 2024-11-21T11:13:43,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:43,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/92c8255f646140969a73db28ffc9d04c is 50, key is test_row_0/A:col10/1732187623387/Put/seqid=0 2024-11-21T11:13:43,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741843_1019 (size=12001) 2024-11-21T11:13:43,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/92c8255f646140969a73db28ffc9d04c 2024-11-21T11:13:43,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:43,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187683424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:43,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:43,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:43,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40396 deadline: 1732187683424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:43,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:43,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:43,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187683429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:43,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:43,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:43,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187683431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:43,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:43,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:43,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187683437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:43,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:43,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T11:13:43,446 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:13:43,447 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:13:43,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-21T11:13:43,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T11:13:43,451 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:13:43,453 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:13:43,453 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:13:43,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/31d051962e5045d280c2d236a0d65c44 is 50, key is test_row_0/B:col10/1732187623387/Put/seqid=0 2024-11-21T11:13:43,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741844_1020 (size=12001) 2024-11-21T11:13:43,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/31d051962e5045d280c2d236a0d65c44 2024-11-21T11:13:43,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/e60244ba77f847aea0349ca4aea057b0 is 50, key is test_row_0/C:col10/1732187623387/Put/seqid=0 2024-11-21T11:13:43,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741845_1021 (size=12001) 2024-11-21T11:13:43,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/e60244ba77f847aea0349ca4aea057b0 2024-11-21T11:13:43,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T11:13:43,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:43,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187683549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:43,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:43,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:43,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187683550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:43,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:43,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:43,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187683549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:43,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/92c8255f646140969a73db28ffc9d04c as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/92c8255f646140969a73db28ffc9d04c 2024-11-21T11:13:43,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:43,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:43,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40396 deadline: 1732187683550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:43,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187683550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:43,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:43,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:43,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:43,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/92c8255f646140969a73db28ffc9d04c, entries=150, sequenceid=51, filesize=11.7 K 2024-11-21T11:13:43,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/31d051962e5045d280c2d236a0d65c44 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/31d051962e5045d280c2d236a0d65c44 2024-11-21T11:13:43,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/31d051962e5045d280c2d236a0d65c44, entries=150, sequenceid=51, filesize=11.7 K 2024-11-21T11:13:43,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/e60244ba77f847aea0349ca4aea057b0 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/e60244ba77f847aea0349ca4aea057b0 2024-11-21T11:13:43,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/e60244ba77f847aea0349ca4aea057b0, entries=150, sequenceid=51, filesize=11.7 K 2024-11-21T11:13:43,606 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for cb944dfe54fbd2ebb234cb2046a89860 in 215ms, sequenceid=51, compaction requested=true 2024-11-21T11:13:43,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:43,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T11:13:43,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:43,608 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing cb944dfe54fbd2ebb234cb2046a89860 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-21T11:13:43,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=A 2024-11-21T11:13:43,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:43,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=B 2024-11-21T11:13:43,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:43,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=C 2024-11-21T11:13:43,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:43,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb944dfe54fbd2ebb234cb2046a89860:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:13:43,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:13:43,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb944dfe54fbd2ebb234cb2046a89860:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:13:43,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:13:43,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb944dfe54fbd2ebb234cb2046a89860:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:13:43,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:13:43,620 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:13:43,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/bfe8bca2c5cd49dba23961818ed5dab1 is 50, key is test_row_0/A:col10/1732187623429/Put/seqid=0 2024-11-21T11:13:43,621 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:13:43,624 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:13:43,624 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:13:43,626 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): cb944dfe54fbd2ebb234cb2046a89860/B is initiating minor compaction (all files) 2024-11-21T11:13:43,627 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb944dfe54fbd2ebb234cb2046a89860/B in TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:43,627 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/8b6f85bef0074899bcb1d517efe1e818, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/fad3b6a8d0b74ab99c4cdf90aa668f83, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/31d051962e5045d280c2d236a0d65c44] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp, totalSize=35.2 K 2024-11-21T11:13:43,629 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b6f85bef0074899bcb1d517efe1e818, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732187622345 2024-11-21T11:13:43,630 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): cb944dfe54fbd2ebb234cb2046a89860/A is initiating minor compaction (all files) 2024-11-21T11:13:43,630 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb944dfe54fbd2ebb234cb2046a89860/A in TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:43,630 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/31ab7a47424249f39fbf9a067fafdf8f, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/4506b49795554c2e951f90a566b3beb8, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/92c8255f646140969a73db28ffc9d04c] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp, totalSize=35.2 K 2024-11-21T11:13:43,631 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting fad3b6a8d0b74ab99c4cdf90aa668f83, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1732187622487 2024-11-21T11:13:43,631 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31ab7a47424249f39fbf9a067fafdf8f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732187622345 2024-11-21T11:13:43,632 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 31d051962e5045d280c2d236a0d65c44, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732187623382 2024-11-21T11:13:43,632 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4506b49795554c2e951f90a566b3beb8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1732187622487 2024-11-21T11:13:43,634 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92c8255f646140969a73db28ffc9d04c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732187623382 2024-11-21T11:13:43,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741846_1022 (size=12001) 2024-11-21T11:13:43,650 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/bfe8bca2c5cd49dba23961818ed5dab1 2024-11-21T11:13:43,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/7ebaa14e3bdd4276b94cb9bd6f2f68ef is 50, key is test_row_0/B:col10/1732187623429/Put/seqid=0 2024-11-21T11:13:43,686 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb944dfe54fbd2ebb234cb2046a89860#A#compaction#10 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:13:43,686 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb944dfe54fbd2ebb234cb2046a89860#B#compaction#11 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:13:43,687 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/bdf34b6149e84c2299fbb4e2d3873276 is 50, key is test_row_0/B:col10/1732187623387/Put/seqid=0 2024-11-21T11:13:43,688 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/7f273322d80649f6b901583db4d7b6e8 is 50, key is test_row_0/A:col10/1732187623387/Put/seqid=0 2024-11-21T11:13:43,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741847_1023 (size=12001) 2024-11-21T11:13:43,725 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/7ebaa14e3bdd4276b94cb9bd6f2f68ef 2024-11-21T11:13:43,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741849_1025 (size=12104) 2024-11-21T11:13:43,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741848_1024 (size=12104) 2024-11-21T11:13:43,762 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/7f273322d80649f6b901583db4d7b6e8 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/7f273322d80649f6b901583db4d7b6e8 2024-11-21T11:13:43,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T11:13:43,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/6c89c2e2b4774f9291341ed4f74b509d is 50, key is test_row_0/C:col10/1732187623429/Put/seqid=0 2024-11-21T11:13:43,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:43,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:13:43,788 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb944dfe54fbd2ebb234cb2046a89860/A of cb944dfe54fbd2ebb234cb2046a89860 into 7f273322d80649f6b901583db4d7b6e8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:13:43,788 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:43,792 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., storeName=cb944dfe54fbd2ebb234cb2046a89860/A, priority=13, startTime=1732187623609; duration=0sec 2024-11-21T11:13:43,792 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:13:43,793 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb944dfe54fbd2ebb234cb2046a89860:A 2024-11-21T11:13:43,793 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:13:43,797 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:13:43,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741850_1026 (size=12001) 2024-11-21T11:13:43,797 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): cb944dfe54fbd2ebb234cb2046a89860/C is initiating minor compaction (all files) 2024-11-21T11:13:43,798 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb944dfe54fbd2ebb234cb2046a89860/C in TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:43,798 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/4f294d546e134601a7125063f955e677, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/812a707b47304e0eb57ef128c28ea027, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/e60244ba77f847aea0349ca4aea057b0] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp, totalSize=35.2 K 2024-11-21T11:13:43,799 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/6c89c2e2b4774f9291341ed4f74b509d 2024-11-21T11:13:43,799 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f294d546e134601a7125063f955e677, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732187622345 2024-11-21T11:13:43,801 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 812a707b47304e0eb57ef128c28ea027, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1732187622487 2024-11-21T11:13:43,803 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting e60244ba77f847aea0349ca4aea057b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732187623382 2024-11-21T11:13:43,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/bfe8bca2c5cd49dba23961818ed5dab1 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/bfe8bca2c5cd49dba23961818ed5dab1 2024-11-21T11:13:43,827 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb944dfe54fbd2ebb234cb2046a89860#C#compaction#14 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:13:43,828 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/9c275bef44604457b639bcc80ab3de98 is 50, key is test_row_0/C:col10/1732187623387/Put/seqid=0 2024-11-21T11:13:43,828 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/bfe8bca2c5cd49dba23961818ed5dab1, entries=150, sequenceid=66, filesize=11.7 K 2024-11-21T11:13:43,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/7ebaa14e3bdd4276b94cb9bd6f2f68ef as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/7ebaa14e3bdd4276b94cb9bd6f2f68ef 2024-11-21T11:13:43,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:43,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187683819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:43,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:43,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187683823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:43,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:43,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187683830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:43,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:43,844 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/7ebaa14e3bdd4276b94cb9bd6f2f68ef, entries=150, sequenceid=66, filesize=11.7 K 2024-11-21T11:13:43,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741851_1027 (size=12104) 2024-11-21T11:13:43,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/6c89c2e2b4774f9291341ed4f74b509d as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/6c89c2e2b4774f9291341ed4f74b509d 2024-11-21T11:13:43,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40396 deadline: 1732187683834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:43,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:43,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:43,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187683841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:43,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:43,858 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/6c89c2e2b4774f9291341ed4f74b509d, entries=150, sequenceid=66, filesize=11.7 K 2024-11-21T11:13:43,860 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for cb944dfe54fbd2ebb234cb2046a89860 in 252ms, sequenceid=66, compaction requested=false 2024-11-21T11:13:43,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:43,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:43,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-21T11:13:43,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-21T11:13:43,870 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-21T11:13:43,870 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 410 msec 2024-11-21T11:13:43,872 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/9c275bef44604457b639bcc80ab3de98 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/9c275bef44604457b639bcc80ab3de98 2024-11-21T11:13:43,875 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 425 msec 2024-11-21T11:13:43,889 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb944dfe54fbd2ebb234cb2046a89860/C of cb944dfe54fbd2ebb234cb2046a89860 into 9c275bef44604457b639bcc80ab3de98(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:13:43,889 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:43,890 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., storeName=cb944dfe54fbd2ebb234cb2046a89860/C, priority=13, startTime=1732187623619; duration=0sec 2024-11-21T11:13:43,890 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:13:43,890 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb944dfe54fbd2ebb234cb2046a89860:C 2024-11-21T11:13:43,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:13:43,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb944dfe54fbd2ebb234cb2046a89860 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T11:13:43,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=A 2024-11-21T11:13:43,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:43,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=B 2024-11-21T11:13:43,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:43,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=C 2024-11-21T11:13:43,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:43,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/06d5796f4c0241a8b295e701ec446f86 is 50, key is test_row_0/A:col10/1732187623830/Put/seqid=0 2024-11-21T11:13:43,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:43,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187683977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:43,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:43,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:43,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40396 deadline: 1732187683979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:43,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741852_1028 (size=16681) 2024-11-21T11:13:43,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:43,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:43,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187683980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:43,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:43,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187683983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:43,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:43,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/06d5796f4c0241a8b295e701ec446f86 2024-11-21T11:13:43,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:43,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187683986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:43,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:43,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:43,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:44,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/4b8f385e2a6149368db5763143a57847 is 50, key is test_row_0/B:col10/1732187623830/Put/seqid=0 2024-11-21T11:13:44,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T11:13:44,076 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:13:44,077 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:13:44,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-21T11:13:44,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T11:13:44,081 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:13:44,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741853_1029 (size=12001) 2024-11-21T11:13:44,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/4b8f385e2a6149368db5763143a57847 2024-11-21T11:13:44,085 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:13:44,085 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:13:44,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:44,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40396 deadline: 1732187684097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:44,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:44,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:44,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:44,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187684097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:44,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187684097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:44,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:44,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:44,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187684105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:44,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:44,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:44,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:44,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187684106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:44,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:44,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/9b6eb1806add4ddcb82d4781e433ced7 is 50, key is test_row_0/C:col10/1732187623830/Put/seqid=0 2024-11-21T11:13:44,169 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/bdf34b6149e84c2299fbb4e2d3873276 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/bdf34b6149e84c2299fbb4e2d3873276 2024-11-21T11:13:44,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741854_1030 (size=12001) 2024-11-21T11:13:44,181 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb944dfe54fbd2ebb234cb2046a89860/B of cb944dfe54fbd2ebb234cb2046a89860 into bdf34b6149e84c2299fbb4e2d3873276(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:13:44,181 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:44,181 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., storeName=cb944dfe54fbd2ebb234cb2046a89860/B, priority=13, startTime=1732187623619; duration=0sec 2024-11-21T11:13:44,181 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:13:44,181 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb944dfe54fbd2ebb234cb2046a89860:B 2024-11-21T11:13:44,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T11:13:44,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T11:13:44,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:44,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:44,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:44,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:44,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:44,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:44,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:44,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40396 deadline: 1732187684307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:44,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187684316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:44,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:44,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:44,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187684326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:44,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:44,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:44,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187684329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:44,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:44,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187684330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:44,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T11:13:44,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T11:13:44,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:44,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:44,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:44,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:44,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:44,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T11:13:44,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:44,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:44,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:44,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:44,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:44,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:44,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/9b6eb1806add4ddcb82d4781e433ced7 2024-11-21T11:13:44,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/06d5796f4c0241a8b295e701ec446f86 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/06d5796f4c0241a8b295e701ec446f86 2024-11-21T11:13:44,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/06d5796f4c0241a8b295e701ec446f86, entries=250, sequenceid=88, filesize=16.3 K 2024-11-21T11:13:44,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/4b8f385e2a6149368db5763143a57847 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/4b8f385e2a6149368db5763143a57847 2024-11-21T11:13:44,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/4b8f385e2a6149368db5763143a57847, entries=150, sequenceid=88, filesize=11.7 K 2024-11-21T11:13:44,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/9b6eb1806add4ddcb82d4781e433ced7 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/9b6eb1806add4ddcb82d4781e433ced7 2024-11-21T11:13:44,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:44,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40396 deadline: 1732187684616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:44,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:44,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/9b6eb1806add4ddcb82d4781e433ced7, entries=150, sequenceid=88, filesize=11.7 K 2024-11-21T11:13:44,640 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for cb944dfe54fbd2ebb234cb2046a89860 in 686ms, sequenceid=88, compaction requested=true 2024-11-21T11:13:44,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:44,641 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:13:44,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb944dfe54fbd2ebb234cb2046a89860:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:13:44,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:13:44,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:13:44,644 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:13:44,645 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:13:44,645 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): cb944dfe54fbd2ebb234cb2046a89860/A is initiating minor compaction (all files) 2024-11-21T11:13:44,645 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb944dfe54fbd2ebb234cb2046a89860/A in TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:44,646 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/7f273322d80649f6b901583db4d7b6e8, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/bfe8bca2c5cd49dba23961818ed5dab1, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/06d5796f4c0241a8b295e701ec446f86] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp, totalSize=39.8 K 2024-11-21T11:13:44,647 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f273322d80649f6b901583db4d7b6e8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732187623382 2024-11-21T11:13:44,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb944dfe54fbd2ebb234cb2046a89860:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:13:44,648 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:13:44,648 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfe8bca2c5cd49dba23961818ed5dab1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732187623412 2024-11-21T11:13:44,649 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): cb944dfe54fbd2ebb234cb2046a89860/B is initiating minor compaction (all files) 2024-11-21T11:13:44,649 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb944dfe54fbd2ebb234cb2046a89860/B in TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:44,649 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/bdf34b6149e84c2299fbb4e2d3873276, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/7ebaa14e3bdd4276b94cb9bd6f2f68ef, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/4b8f385e2a6149368db5763143a57847] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp, totalSize=35.3 K 2024-11-21T11:13:44,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:13:44,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb944dfe54fbd2ebb234cb2046a89860:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:13:44,649 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06d5796f4c0241a8b295e701ec446f86, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732187623808 2024-11-21T11:13:44,650 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting bdf34b6149e84c2299fbb4e2d3873276, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732187623382 2024-11-21T11:13:44,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:13:44,650 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb944dfe54fbd2ebb234cb2046a89860 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-21T11:13:44,651 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ebaa14e3bdd4276b94cb9bd6f2f68ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732187623412 2024-11-21T11:13:44,652 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b8f385e2a6149368db5763143a57847, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732187623808 2024-11-21T11:13:44,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=A 2024-11-21T11:13:44,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:44,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=B 2024-11-21T11:13:44,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:44,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=C 2024-11-21T11:13:44,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:44,681 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb944dfe54fbd2ebb234cb2046a89860#A#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:13:44,682 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/c3b2f1d6b5fb425199acd349ecec76b2 is 50, key is test_row_0/A:col10/1732187623830/Put/seqid=0 2024-11-21T11:13:44,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/15c6ac36db174b04bb3167f23d1618f1 is 50, key is test_row_0/A:col10/1732187624640/Put/seqid=0 2024-11-21T11:13:44,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T11:13:44,707 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb944dfe54fbd2ebb234cb2046a89860#B#compaction#20 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:13:44,708 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/7eaec5076bf84b758e9515c6e6046aeb is 50, key is test_row_0/B:col10/1732187623830/Put/seqid=0 2024-11-21T11:13:44,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741855_1031 (size=12207) 2024-11-21T11:13:44,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T11:13:44,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:44,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:44,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:44,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:44,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:44,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:44,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741856_1032 (size=12001) 2024-11-21T11:13:44,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/15c6ac36db174b04bb3167f23d1618f1 2024-11-21T11:13:44,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741857_1033 (size=12207) 2024-11-21T11:13:44,735 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/c3b2f1d6b5fb425199acd349ecec76b2 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/c3b2f1d6b5fb425199acd349ecec76b2 2024-11-21T11:13:44,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:44,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187684727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:44,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:44,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:44,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187684732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:44,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:44,747 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/7eaec5076bf84b758e9515c6e6046aeb as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/7eaec5076bf84b758e9515c6e6046aeb 2024-11-21T11:13:44,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/e398bf2579404175a74ffb4126afccab is 50, key is test_row_0/B:col10/1732187624640/Put/seqid=0 2024-11-21T11:13:44,750 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb944dfe54fbd2ebb234cb2046a89860/A of cb944dfe54fbd2ebb234cb2046a89860 into c3b2f1d6b5fb425199acd349ecec76b2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:13:44,751 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:44,751 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., storeName=cb944dfe54fbd2ebb234cb2046a89860/A, priority=13, startTime=1732187624640; duration=0sec 2024-11-21T11:13:44,752 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:13:44,752 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb944dfe54fbd2ebb234cb2046a89860:A 2024-11-21T11:13:44,752 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:13:44,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:44,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187684737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:44,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,756 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:13:44,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:44,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:44,756 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): cb944dfe54fbd2ebb234cb2046a89860/C is initiating minor compaction (all files) 2024-11-21T11:13:44,756 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb944dfe54fbd2ebb234cb2046a89860/C in TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:44,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187684742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:44,756 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/9c275bef44604457b639bcc80ab3de98, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/6c89c2e2b4774f9291341ed4f74b509d, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/9b6eb1806add4ddcb82d4781e433ced7] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp, totalSize=35.3 K 2024-11-21T11:13:44,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:44,757 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c275bef44604457b639bcc80ab3de98, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732187623382 2024-11-21T11:13:44,758 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c89c2e2b4774f9291341ed4f74b509d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732187623412 2024-11-21T11:13:44,760 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b6eb1806add4ddcb82d4781e433ced7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732187623808 2024-11-21T11:13:44,765 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb944dfe54fbd2ebb234cb2046a89860/B of cb944dfe54fbd2ebb234cb2046a89860 into 7eaec5076bf84b758e9515c6e6046aeb(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:13:44,765 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:44,765 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., storeName=cb944dfe54fbd2ebb234cb2046a89860/B, priority=13, startTime=1732187624644; duration=0sec 2024-11-21T11:13:44,765 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:13:44,765 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb944dfe54fbd2ebb234cb2046a89860:B 2024-11-21T11:13:44,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741858_1034 (size=12001) 2024-11-21T11:13:44,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/e398bf2579404175a74ffb4126afccab 2024-11-21T11:13:44,800 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb944dfe54fbd2ebb234cb2046a89860#C#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:13:44,801 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/8e7887363e914ac2b0b6902d58c1762c is 50, key is test_row_0/C:col10/1732187623830/Put/seqid=0 2024-11-21T11:13:44,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/12231413f31f41148d654cef33302d23 is 50, key is test_row_0/C:col10/1732187624640/Put/seqid=0 2024-11-21T11:13:44,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:44,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187684846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:44,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:44,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:44,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187684846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:44,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:44,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741859_1035 (size=12207) 2024-11-21T11:13:44,864 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/8e7887363e914ac2b0b6902d58c1762c as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/8e7887363e914ac2b0b6902d58c1762c 2024-11-21T11:13:44,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:44,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187684866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:44,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T11:13:44,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:44,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:44,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:44,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:44,875 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:44,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:44,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:44,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187684866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:44,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:44,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:44,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741860_1036 (size=12001) 2024-11-21T11:13:44,877 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb944dfe54fbd2ebb234cb2046a89860/C of cb944dfe54fbd2ebb234cb2046a89860 into 8e7887363e914ac2b0b6902d58c1762c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:13:44,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:44,877 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:44,878 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., storeName=cb944dfe54fbd2ebb234cb2046a89860/C, priority=13, startTime=1732187624649; duration=0sec 2024-11-21T11:13:44,878 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:13:44,878 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb944dfe54fbd2ebb234cb2046a89860:C 2024-11-21T11:13:45,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T11:13:45,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:45,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:45,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:45,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:45,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:45,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:45,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:45,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187685056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:45,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:45,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187685056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:45,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:45,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:45,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:45,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:45,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:45,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:45,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:45,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187685086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:45,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:45,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:45,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:45,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:45,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187685086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:45,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:45,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:45,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:45,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:45,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40396 deadline: 1732187685125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:45,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:45,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:45,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:45,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T11:13:45,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:45,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:45,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:45,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:45,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:45,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:45,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T11:13:45,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/12231413f31f41148d654cef33302d23 2024-11-21T11:13:45,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/15c6ac36db174b04bb3167f23d1618f1 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/15c6ac36db174b04bb3167f23d1618f1 2024-11-21T11:13:45,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/15c6ac36db174b04bb3167f23d1618f1, entries=150, sequenceid=102, filesize=11.7 K 2024-11-21T11:13:45,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/e398bf2579404175a74ffb4126afccab as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/e398bf2579404175a74ffb4126afccab 2024-11-21T11:13:45,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/e398bf2579404175a74ffb4126afccab, entries=150, sequenceid=102, filesize=11.7 K 2024-11-21T11:13:45,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/12231413f31f41148d654cef33302d23 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/12231413f31f41148d654cef33302d23 2024-11-21T11:13:45,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/12231413f31f41148d654cef33302d23, entries=150, sequenceid=102, filesize=11.7 K 2024-11-21T11:13:45,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for cb944dfe54fbd2ebb234cb2046a89860 in 677ms, sequenceid=102, compaction requested=false 2024-11-21T11:13:45,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:45,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T11:13:45,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:45,344 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing cb944dfe54fbd2ebb234cb2046a89860 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-21T11:13:45,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=A 2024-11-21T11:13:45,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:45,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=B 2024-11-21T11:13:45,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:45,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=C 2024-11-21T11:13:45,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:45,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/15235ed32b464296a58efe255df2c0b7 is 50, key is test_row_0/A:col10/1732187624723/Put/seqid=0 2024-11-21T11:13:45,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:45,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:13:45,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741861_1037 (size=12001) 2024-11-21T11:13:45,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:45,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187685410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:45,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:45,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:45,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:45,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:45,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187685412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:45,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:45,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:45,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:45,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:45,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187685412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:45,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:45,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:45,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187685412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:45,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:45,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:45,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:45,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:45,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:45,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:45,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187685515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:45,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:45,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:45,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:45,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:45,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187685526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:45,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:45,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187685526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:45,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:45,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187685527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:45,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:45,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:45,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:45,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:45,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187685736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:45,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:45,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:45,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187685736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:45,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:45,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:45,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:45,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:45,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:45,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:45,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187685748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:45,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:45,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:45,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:45,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:45,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187685748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:45,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:45,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:45,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:45,789 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/15235ed32b464296a58efe255df2c0b7 2024-11-21T11:13:45,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/b64702f55cd04b6c8f701e6c222cacac is 50, key is test_row_0/B:col10/1732187624723/Put/seqid=0 2024-11-21T11:13:45,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741862_1038 (size=12001) 2024-11-21T11:13:46,034 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T11:13:46,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:46,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187686056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:46,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:46,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:46,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:46,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:46,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187686056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:46,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:46,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:46,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:46,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:46,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187686066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:46,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:46,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:46,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:46,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:46,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187686056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:46,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:46,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:46,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:46,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:46,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40396 deadline: 1732187686153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:46,159 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-21T11:13:46,160 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_1' in cb944dfe54fbd2ebb234cb2046a89860 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:13:46,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:46,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:46,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:46,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T11:13:46,241 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/b64702f55cd04b6c8f701e6c222cacac 2024-11-21T11:13:46,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/6e232429fd6947049f9c2392061baf04 is 50, key is test_row_0/C:col10/1732187624723/Put/seqid=0 2024-11-21T11:13:46,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741863_1039 (size=12001) 2024-11-21T11:13:46,297 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/6e232429fd6947049f9c2392061baf04 2024-11-21T11:13:46,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/15235ed32b464296a58efe255df2c0b7 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/15235ed32b464296a58efe255df2c0b7 2024-11-21T11:13:46,325 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/15235ed32b464296a58efe255df2c0b7, entries=150, sequenceid=125, filesize=11.7 K 2024-11-21T11:13:46,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/b64702f55cd04b6c8f701e6c222cacac as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/b64702f55cd04b6c8f701e6c222cacac 2024-11-21T11:13:46,343 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/b64702f55cd04b6c8f701e6c222cacac, entries=150, sequenceid=125, filesize=11.7 K 2024-11-21T11:13:46,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/6e232429fd6947049f9c2392061baf04 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/6e232429fd6947049f9c2392061baf04 2024-11-21T11:13:46,361 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/6e232429fd6947049f9c2392061baf04, entries=150, sequenceid=125, filesize=11.7 K 2024-11-21T11:13:46,364 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for cb944dfe54fbd2ebb234cb2046a89860 in 1020ms, sequenceid=125, compaction requested=true 2024-11-21T11:13:46,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:46,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:46,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-21T11:13:46,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-21T11:13:46,373 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-21T11:13:46,373 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2820 sec 2024-11-21T11:13:46,377 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.2960 sec 2024-11-21T11:13:46,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:13:46,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb944dfe54fbd2ebb234cb2046a89860 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T11:13:46,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=A 2024-11-21T11:13:46,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:46,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=B 2024-11-21T11:13:46,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:46,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=C 2024-11-21T11:13:46,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:46,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/c0617618cdc945c799b3aee8fcb81e16 is 50, key is test_row_0/A:col10/1732187625406/Put/seqid=0 2024-11-21T11:13:46,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741864_1040 (size=14541) 2024-11-21T11:13:46,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/c0617618cdc945c799b3aee8fcb81e16 2024-11-21T11:13:46,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/112f3c12b8034d378535a5c37dce88eb is 50, key is test_row_0/B:col10/1732187625406/Put/seqid=0 2024-11-21T11:13:46,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:46,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187686655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:46,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:46,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:46,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:46,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:46,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187686655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:46,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:46,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:46,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:46,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:46,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187686657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:46,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:46,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:46,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:46,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:46,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187686660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:46,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:46,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:46,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:46,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741865_1041 (size=12151) 2024-11-21T11:13:46,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/112f3c12b8034d378535a5c37dce88eb 2024-11-21T11:13:46,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/6010738385404ec6a905a0d2222e4a9d is 50, key is test_row_0/C:col10/1732187625406/Put/seqid=0 2024-11-21T11:13:46,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741866_1042 (size=12151) 2024-11-21T11:13:46,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:46,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187686776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:46,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:46,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:46,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187686776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:46,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187686776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:46,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:46,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:46,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:46,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:46,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:46,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:46,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:46,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187686777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:46,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:46,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:46,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:46,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:46,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:46,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:46,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:46,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187686986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:46,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:46,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:46,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:46,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:46,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:46,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187686986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:46,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187686986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:46,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:46,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:46,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:46,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:46,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:46,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:46,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:46,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187686986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:46,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:46,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:46,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:47,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/6010738385404ec6a905a0d2222e4a9d 2024-11-21T11:13:47,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/c0617618cdc945c799b3aee8fcb81e16 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/c0617618cdc945c799b3aee8fcb81e16 2024-11-21T11:13:47,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/c0617618cdc945c799b3aee8fcb81e16, entries=200, sequenceid=139, filesize=14.2 K 2024-11-21T11:13:47,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/112f3c12b8034d378535a5c37dce88eb as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/112f3c12b8034d378535a5c37dce88eb 2024-11-21T11:13:47,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/112f3c12b8034d378535a5c37dce88eb, entries=150, sequenceid=139, filesize=11.9 K 2024-11-21T11:13:47,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/6010738385404ec6a905a0d2222e4a9d as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/6010738385404ec6a905a0d2222e4a9d 2024-11-21T11:13:47,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/6010738385404ec6a905a0d2222e4a9d, entries=150, sequenceid=139, filesize=11.9 K 2024-11-21T11:13:47,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for cb944dfe54fbd2ebb234cb2046a89860 in 594ms, sequenceid=139, compaction requested=true 2024-11-21T11:13:47,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:47,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb944dfe54fbd2ebb234cb2046a89860:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:13:47,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:13:47,164 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T11:13:47,164 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T11:13:47,167 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T11:13:47,167 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T11:13:47,167 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): cb944dfe54fbd2ebb234cb2046a89860/A is initiating minor compaction (all files) 2024-11-21T11:13:47,167 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): cb944dfe54fbd2ebb234cb2046a89860/B is initiating minor compaction (all files) 2024-11-21T11:13:47,167 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb944dfe54fbd2ebb234cb2046a89860/A in TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:47,167 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb944dfe54fbd2ebb234cb2046a89860/B in TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:47,167 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/7eaec5076bf84b758e9515c6e6046aeb, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/e398bf2579404175a74ffb4126afccab, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/b64702f55cd04b6c8f701e6c222cacac, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/112f3c12b8034d378535a5c37dce88eb] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp, totalSize=47.2 K 2024-11-21T11:13:47,167 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/c3b2f1d6b5fb425199acd349ecec76b2, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/15c6ac36db174b04bb3167f23d1618f1, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/15235ed32b464296a58efe255df2c0b7, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/c0617618cdc945c799b3aee8fcb81e16] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp, totalSize=49.6 K 2024-11-21T11:13:47,168 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 7eaec5076bf84b758e9515c6e6046aeb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732187623808 2024-11-21T11:13:47,168 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3b2f1d6b5fb425199acd349ecec76b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732187623808 2024-11-21T11:13:47,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb944dfe54fbd2ebb234cb2046a89860:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:13:47,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:13:47,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb944dfe54fbd2ebb234cb2046a89860:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:13:47,169 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting e398bf2579404175a74ffb4126afccab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732187623961 2024-11-21T11:13:47,169 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15c6ac36db174b04bb3167f23d1618f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732187623961 2024-11-21T11:13:47,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:13:47,170 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15235ed32b464296a58efe255df2c0b7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1732187624723 2024-11-21T11:13:47,170 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting b64702f55cd04b6c8f701e6c222cacac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1732187624723 2024-11-21T11:13:47,170 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0617618cdc945c799b3aee8fcb81e16, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732187625406 2024-11-21T11:13:47,171 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 112f3c12b8034d378535a5c37dce88eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732187625406 2024-11-21T11:13:47,205 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb944dfe54fbd2ebb234cb2046a89860#A#compaction#30 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:13:47,206 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/5058748b6aab4498ac05d10dc4568d13 is 50, key is test_row_0/A:col10/1732187625406/Put/seqid=0 2024-11-21T11:13:47,215 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb944dfe54fbd2ebb234cb2046a89860#B#compaction#31 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:13:47,216 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/4fed60baacfb4835bd083fcc37b44b5b is 50, key is test_row_0/B:col10/1732187625406/Put/seqid=0 2024-11-21T11:13:47,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741867_1043 (size=12493) 2024-11-21T11:13:47,239 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/5058748b6aab4498ac05d10dc4568d13 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/5058748b6aab4498ac05d10dc4568d13 2024-11-21T11:13:47,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741868_1044 (size=12493) 2024-11-21T11:13:47,251 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cb944dfe54fbd2ebb234cb2046a89860/A of cb944dfe54fbd2ebb234cb2046a89860 into 5058748b6aab4498ac05d10dc4568d13(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:13:47,251 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:47,251 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., storeName=cb944dfe54fbd2ebb234cb2046a89860/A, priority=12, startTime=1732187627164; duration=0sec 2024-11-21T11:13:47,251 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:13:47,251 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb944dfe54fbd2ebb234cb2046a89860:A 2024-11-21T11:13:47,251 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T11:13:47,254 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T11:13:47,255 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): cb944dfe54fbd2ebb234cb2046a89860/C is initiating minor compaction (all files) 2024-11-21T11:13:47,256 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb944dfe54fbd2ebb234cb2046a89860/C in TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:47,257 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/8e7887363e914ac2b0b6902d58c1762c, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/12231413f31f41148d654cef33302d23, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/6e232429fd6947049f9c2392061baf04, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/6010738385404ec6a905a0d2222e4a9d] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp, totalSize=47.2 K 2024-11-21T11:13:47,259 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e7887363e914ac2b0b6902d58c1762c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732187623808 2024-11-21T11:13:47,259 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12231413f31f41148d654cef33302d23, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732187623961 2024-11-21T11:13:47,261 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e232429fd6947049f9c2392061baf04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1732187624723 2024-11-21T11:13:47,262 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6010738385404ec6a905a0d2222e4a9d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732187625406 2024-11-21T11:13:47,265 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/4fed60baacfb4835bd083fcc37b44b5b as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/4fed60baacfb4835bd083fcc37b44b5b 2024-11-21T11:13:47,275 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cb944dfe54fbd2ebb234cb2046a89860/B of cb944dfe54fbd2ebb234cb2046a89860 into 4fed60baacfb4835bd083fcc37b44b5b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:13:47,275 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:47,275 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., storeName=cb944dfe54fbd2ebb234cb2046a89860/B, priority=12, startTime=1732187627164; duration=0sec 2024-11-21T11:13:47,275 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:13:47,275 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb944dfe54fbd2ebb234cb2046a89860:B 2024-11-21T11:13:47,295 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb944dfe54fbd2ebb234cb2046a89860#C#compaction#32 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:13:47,296 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/780be62f012d44d8965c679586cc56b7 is 50, key is test_row_0/C:col10/1732187625406/Put/seqid=0 2024-11-21T11:13:47,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:13:47,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb944dfe54fbd2ebb234cb2046a89860 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T11:13:47,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=A 2024-11-21T11:13:47,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:47,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=B 2024-11-21T11:13:47,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:47,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=C 2024-11-21T11:13:47,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:47,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/c0ec46da75be4713b194e727d7b864e3 is 50, key is test_row_0/A:col10/1732187627298/Put/seqid=0 2024-11-21T11:13:47,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741869_1045 (size=12493) 2024-11-21T11:13:47,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741870_1046 (size=14541) 2024-11-21T11:13:47,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/c0ec46da75be4713b194e727d7b864e3 2024-11-21T11:13:47,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:47,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187687344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:47,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:47,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/fb47550262304a40b79ff78279ab0881 is 50, key is test_row_0/B:col10/1732187627298/Put/seqid=0 2024-11-21T11:13:47,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:47,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:47,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:47,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187687351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:47,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:47,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187687344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:47,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:47,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:47,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:47,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:47,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187687352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:47,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:47,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:47,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:47,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:47,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:47,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:47,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741871_1047 (size=12151) 2024-11-21T11:13:47,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:47,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187687466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:47,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:47,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:47,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:47,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:47,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187687466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:47,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:47,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:47,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187687466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:47,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:47,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:47,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:47,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:47,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:47,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:47,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187687466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:47,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:47,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:47,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:47,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187687677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:47,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:47,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187687677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:47,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:47,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:47,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187687677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:47,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:47,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:47,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:47,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187687679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:47,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:47,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:47,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:47,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:47,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:47,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:47,747 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/780be62f012d44d8965c679586cc56b7 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/780be62f012d44d8965c679586cc56b7 2024-11-21T11:13:47,761 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cb944dfe54fbd2ebb234cb2046a89860/C of cb944dfe54fbd2ebb234cb2046a89860 into 780be62f012d44d8965c679586cc56b7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:13:47,761 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:47,761 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., storeName=cb944dfe54fbd2ebb234cb2046a89860/C, priority=12, startTime=1732187627169; duration=0sec 2024-11-21T11:13:47,761 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:13:47,761 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb944dfe54fbd2ebb234cb2046a89860:C 2024-11-21T11:13:47,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/fb47550262304a40b79ff78279ab0881 2024-11-21T11:13:47,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/57e82db449fa456b9aa50abe9aae3942 is 50, key is test_row_0/C:col10/1732187627298/Put/seqid=0 2024-11-21T11:13:47,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741872_1048 (size=12151) 2024-11-21T11:13:47,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/57e82db449fa456b9aa50abe9aae3942 2024-11-21T11:13:47,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/c0ec46da75be4713b194e727d7b864e3 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/c0ec46da75be4713b194e727d7b864e3 2024-11-21T11:13:47,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/c0ec46da75be4713b194e727d7b864e3, entries=200, sequenceid=160, filesize=14.2 K 2024-11-21T11:13:47,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/fb47550262304a40b79ff78279ab0881 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/fb47550262304a40b79ff78279ab0881 2024-11-21T11:13:47,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/fb47550262304a40b79ff78279ab0881, entries=150, sequenceid=160, filesize=11.9 K 2024-11-21T11:13:47,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/57e82db449fa456b9aa50abe9aae3942 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/57e82db449fa456b9aa50abe9aae3942 2024-11-21T11:13:47,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/57e82db449fa456b9aa50abe9aae3942, entries=150, sequenceid=160, filesize=11.9 K 2024-11-21T11:13:47,882 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for cb944dfe54fbd2ebb234cb2046a89860 in 581ms, sequenceid=160, compaction requested=false 2024-11-21T11:13:47,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:47,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:13:47,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb944dfe54fbd2ebb234cb2046a89860 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T11:13:47,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=A 2024-11-21T11:13:47,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:47,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=B 2024-11-21T11:13:47,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:47,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=C 2024-11-21T11:13:47,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:48,004 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-21T11:13:48,006 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-21T11:13:48,008 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T11:13:48,008 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-21T11:13:48,009 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-21T11:13:48,009 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-21T11:13:48,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/60d23b4b7a3b498791770db943e040c0 is 50, key is test_row_0/A:col10/1732187627989/Put/seqid=0 2024-11-21T11:13:48,010 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-21T11:13:48,010 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-21T11:13:48,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741873_1049 (size=16931) 2024-11-21T11:13:48,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:48,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:48,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187688036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:48,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187688039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:48,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:48,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187688043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:48,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:48,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:48,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:48,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:48,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187688046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:48,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:48,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:48,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:48,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:48,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187688155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:48,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:48,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:48,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:48,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187688156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:48,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:48,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:48,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:48,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:48,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:48,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187688156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:48,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:48,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:48,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:48,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:48,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187688165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:48,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:48,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:48,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:48,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:48,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40396 deadline: 1732187688186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:48,191 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_1' in cb944dfe54fbd2ebb234cb2046a89860 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:13:48,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:48,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:48,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:48,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T11:13:48,246 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:13:48,247 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:13:48,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-21T11:13:48,251 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:13:48,252 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:13:48,252 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:13:48,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T11:13:48,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T11:13:48,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:48,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187688365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:48,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:48,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187688366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:48,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:48,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:48,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:48,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:48,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187688367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:48,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:48,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:48,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:48,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:48,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187688376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:48,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:48,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:48,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:48,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T11:13:48,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:48,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:48,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:48,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:48,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:48,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:48,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/60d23b4b7a3b498791770db943e040c0 2024-11-21T11:13:48,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/3e24b13c1d604f40897f246db374ad70 is 50, key is test_row_0/B:col10/1732187627989/Put/seqid=0 2024-11-21T11:13:48,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741874_1050 (size=12151) 2024-11-21T11:13:48,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T11:13:48,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:48,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:48,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:48,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:48,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:48,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:48,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T11:13:48,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:48,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187688676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:48,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:48,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:48,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:48,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:48,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187688676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:48,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:48,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187688676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:48,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:48,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:48,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:48,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:48,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187688685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:48,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:48,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:48,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:48,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T11:13:48,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:48,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:48,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:48,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:48,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:48,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:48,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T11:13:48,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:48,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:48,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:48,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:48,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:48,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:48,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T11:13:48,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/3e24b13c1d604f40897f246db374ad70 2024-11-21T11:13:48,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/2741522c508b4efe9613e52aebe5cba2 is 50, key is test_row_0/C:col10/1732187627989/Put/seqid=0 2024-11-21T11:13:48,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741875_1051 (size=12151) 2024-11-21T11:13:48,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/2741522c508b4efe9613e52aebe5cba2 2024-11-21T11:13:48,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/60d23b4b7a3b498791770db943e040c0 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/60d23b4b7a3b498791770db943e040c0 2024-11-21T11:13:48,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/60d23b4b7a3b498791770db943e040c0, entries=250, sequenceid=176, filesize=16.5 K 2024-11-21T11:13:48,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/3e24b13c1d604f40897f246db374ad70 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/3e24b13c1d604f40897f246db374ad70 2024-11-21T11:13:48,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/3e24b13c1d604f40897f246db374ad70, entries=150, sequenceid=176, filesize=11.9 K 2024-11-21T11:13:48,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/2741522c508b4efe9613e52aebe5cba2 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/2741522c508b4efe9613e52aebe5cba2 2024-11-21T11:13:49,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/2741522c508b4efe9613e52aebe5cba2, entries=150, sequenceid=176, filesize=11.9 K 2024-11-21T11:13:49,010 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for cb944dfe54fbd2ebb234cb2046a89860 in 1020ms, sequenceid=176, compaction requested=true 2024-11-21T11:13:49,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:49,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb944dfe54fbd2ebb234cb2046a89860:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:13:49,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:13:49,010 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:13:49,010 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:13:49,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb944dfe54fbd2ebb234cb2046a89860:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:13:49,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:13:49,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb944dfe54fbd2ebb234cb2046a89860:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:13:49,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:13:49,012 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:13:49,012 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:13:49,013 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): cb944dfe54fbd2ebb234cb2046a89860/A is initiating minor compaction (all files) 2024-11-21T11:13:49,013 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): cb944dfe54fbd2ebb234cb2046a89860/B is initiating minor compaction (all files) 2024-11-21T11:13:49,013 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb944dfe54fbd2ebb234cb2046a89860/A in TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:49,013 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb944dfe54fbd2ebb234cb2046a89860/B in TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:49,013 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/5058748b6aab4498ac05d10dc4568d13, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/c0ec46da75be4713b194e727d7b864e3, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/60d23b4b7a3b498791770db943e040c0] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp, totalSize=42.9 K 2024-11-21T11:13:49,013 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/4fed60baacfb4835bd083fcc37b44b5b, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/fb47550262304a40b79ff78279ab0881, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/3e24b13c1d604f40897f246db374ad70] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp, totalSize=35.9 K 2024-11-21T11:13:49,014 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5058748b6aab4498ac05d10dc4568d13, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732187625406 2024-11-21T11:13:49,015 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fed60baacfb4835bd083fcc37b44b5b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732187625406 2024-11-21T11:13:49,017 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0ec46da75be4713b194e727d7b864e3, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732187626600 2024-11-21T11:13:49,017 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting fb47550262304a40b79ff78279ab0881, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732187626600 2024-11-21T11:13:49,017 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60d23b4b7a3b498791770db943e040c0, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732187627323 2024-11-21T11:13:49,018 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e24b13c1d604f40897f246db374ad70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732187627323 2024-11-21T11:13:49,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T11:13:49,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:49,026 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing cb944dfe54fbd2ebb234cb2046a89860 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-21T11:13:49,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=A 2024-11-21T11:13:49,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:49,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=B 2024-11-21T11:13:49,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:49,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=C 2024-11-21T11:13:49,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:49,043 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb944dfe54fbd2ebb234cb2046a89860#A#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:13:49,044 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/b87f05ef8332415eabf861593aca33f6 is 50, key is test_row_0/A:col10/1732187627989/Put/seqid=0 2024-11-21T11:13:49,048 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb944dfe54fbd2ebb234cb2046a89860#B#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:13:49,049 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/9aebd59358764fd7bdf53eb86d1cecc1 is 50, key is test_row_0/B:col10/1732187627989/Put/seqid=0 2024-11-21T11:13:49,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/2c0a563e77b2415d8a5ca25e5318b63d is 50, key is test_row_0/A:col10/1732187627999/Put/seqid=0 2024-11-21T11:13:49,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741876_1052 (size=12595) 2024-11-21T11:13:49,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741878_1054 (size=12151) 2024-11-21T11:13:49,084 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/2c0a563e77b2415d8a5ca25e5318b63d 2024-11-21T11:13:49,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741877_1053 (size=12595) 2024-11-21T11:13:49,103 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/9aebd59358764fd7bdf53eb86d1cecc1 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/9aebd59358764fd7bdf53eb86d1cecc1 2024-11-21T11:13:49,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/9ef407c96a774fce8ee62a170525a518 is 50, key is test_row_0/B:col10/1732187627999/Put/seqid=0 2024-11-21T11:13:49,117 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb944dfe54fbd2ebb234cb2046a89860/B of cb944dfe54fbd2ebb234cb2046a89860 into 9aebd59358764fd7bdf53eb86d1cecc1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:13:49,117 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:49,117 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., storeName=cb944dfe54fbd2ebb234cb2046a89860/B, priority=13, startTime=1732187629010; duration=0sec 2024-11-21T11:13:49,117 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:13:49,117 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb944dfe54fbd2ebb234cb2046a89860:B 2024-11-21T11:13:49,117 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:13:49,119 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:13:49,119 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): cb944dfe54fbd2ebb234cb2046a89860/C is initiating minor compaction (all files) 2024-11-21T11:13:49,119 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb944dfe54fbd2ebb234cb2046a89860/C in TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:49,119 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/780be62f012d44d8965c679586cc56b7, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/57e82db449fa456b9aa50abe9aae3942, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/2741522c508b4efe9613e52aebe5cba2] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp, totalSize=35.9 K 2024-11-21T11:13:49,120 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 780be62f012d44d8965c679586cc56b7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732187625406 2024-11-21T11:13:49,121 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 57e82db449fa456b9aa50abe9aae3942, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732187626600 2024-11-21T11:13:49,121 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 2741522c508b4efe9613e52aebe5cba2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732187627323 2024-11-21T11:13:49,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741879_1055 (size=12151) 2024-11-21T11:13:49,147 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb944dfe54fbd2ebb234cb2046a89860#C#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:13:49,148 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/5fff6e08bb644e7296422260088b9c7e is 50, key is test_row_0/C:col10/1732187627989/Put/seqid=0 2024-11-21T11:13:49,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741880_1056 (size=12595) 2024-11-21T11:13:49,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:49,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:13:49,191 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/5fff6e08bb644e7296422260088b9c7e as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/5fff6e08bb644e7296422260088b9c7e 2024-11-21T11:13:49,200 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb944dfe54fbd2ebb234cb2046a89860/C of cb944dfe54fbd2ebb234cb2046a89860 into 5fff6e08bb644e7296422260088b9c7e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:13:49,201 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:49,201 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., storeName=cb944dfe54fbd2ebb234cb2046a89860/C, priority=13, startTime=1732187629011; duration=0sec 2024-11-21T11:13:49,202 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:13:49,202 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb944dfe54fbd2ebb234cb2046a89860:C 2024-11-21T11:13:49,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:49,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187689214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:49,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:49,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187689215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:49,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:49,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187689217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:49,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:49,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:49,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:49,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187689223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:49,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:49,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:49,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187689335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:49,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:49,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:49,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:49,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187689336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:49,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:49,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:49,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:49,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:49,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:49,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187689336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:49,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:49,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:49,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:49,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:49,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187689336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:49,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T11:13:49,489 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/b87f05ef8332415eabf861593aca33f6 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/b87f05ef8332415eabf861593aca33f6 2024-11-21T11:13:49,500 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb944dfe54fbd2ebb234cb2046a89860/A of cb944dfe54fbd2ebb234cb2046a89860 into b87f05ef8332415eabf861593aca33f6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:13:49,500 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:49,500 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., storeName=cb944dfe54fbd2ebb234cb2046a89860/A, priority=13, startTime=1732187629010; duration=0sec 2024-11-21T11:13:49,500 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:13:49,500 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb944dfe54fbd2ebb234cb2046a89860:A 2024-11-21T11:13:49,536 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/9ef407c96a774fce8ee62a170525a518 2024-11-21T11:13:49,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:49,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187689547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:49,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:49,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:49,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:49,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:49,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187689547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:49,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:49,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:49,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:49,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:49,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187689547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:49,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:49,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:49,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:49,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:49,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187689547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:49,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/2b3ef9e752ca42f5b4acd393cabd67e6 is 50, key is test_row_0/C:col10/1732187627999/Put/seqid=0 2024-11-21T11:13:49,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:49,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:49,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:49,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741881_1057 (size=12151) 2024-11-21T11:13:49,562 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/2b3ef9e752ca42f5b4acd393cabd67e6 2024-11-21T11:13:49,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/2c0a563e77b2415d8a5ca25e5318b63d as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/2c0a563e77b2415d8a5ca25e5318b63d 2024-11-21T11:13:49,586 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/2c0a563e77b2415d8a5ca25e5318b63d, entries=150, sequenceid=193, filesize=11.9 K 2024-11-21T11:13:49,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/9ef407c96a774fce8ee62a170525a518 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/9ef407c96a774fce8ee62a170525a518 2024-11-21T11:13:49,596 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/9ef407c96a774fce8ee62a170525a518, entries=150, sequenceid=193, filesize=11.9 K 2024-11-21T11:13:49,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/2b3ef9e752ca42f5b4acd393cabd67e6 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/2b3ef9e752ca42f5b4acd393cabd67e6 2024-11-21T11:13:49,603 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/2b3ef9e752ca42f5b4acd393cabd67e6, entries=150, sequenceid=193, filesize=11.9 K 2024-11-21T11:13:49,608 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for cb944dfe54fbd2ebb234cb2046a89860 in 582ms, sequenceid=193, compaction requested=false 2024-11-21T11:13:49,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:49,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:49,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-21T11:13:49,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-21T11:13:49,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-21T11:13:49,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3590 sec 2024-11-21T11:13:49,617 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.3680 sec 2024-11-21T11:13:49,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:13:49,859 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb944dfe54fbd2ebb234cb2046a89860 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-21T11:13:49,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=A 2024-11-21T11:13:49,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:49,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=B 2024-11-21T11:13:49,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:49,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=C 2024-11-21T11:13:49,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:49,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/0577bfc3aa434c42b0ecd0c5a4fe1cab is 50, key is test_row_0/A:col10/1732187629857/Put/seqid=0 2024-11-21T11:13:49,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741882_1058 (size=14541) 2024-11-21T11:13:49,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/0577bfc3aa434c42b0ecd0c5a4fe1cab 2024-11-21T11:13:49,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:49,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187689892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:49,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:49,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:49,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:49,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:49,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187689892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:49,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:49,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:49,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:49,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:49,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187689894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:49,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:49,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:49,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:49,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:49,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187689897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:49,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:49,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:49,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:49,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/c9ee4ca2c333450e91e8eb32ea874f14 is 50, key is test_row_0/B:col10/1732187629857/Put/seqid=0 2024-11-21T11:13:49,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741883_1059 (size=12151) 2024-11-21T11:13:50,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:50,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187690006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:50,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:50,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187690006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:50,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:50,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:50,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:50,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187690016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:50,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:50,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:50,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187690016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:50,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:50,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:50,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187690215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:50,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:50,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187690216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:50,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:50,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:50,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:50,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187690225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:50,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:50,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187690225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:50,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:50,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:50,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/c9ee4ca2c333450e91e8eb32ea874f14 2024-11-21T11:13:50,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/b22e6aeccdfb4362b8ac7e7608c435d7 is 50, key is test_row_0/C:col10/1732187629857/Put/seqid=0 2024-11-21T11:13:50,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741884_1060 (size=12151) 2024-11-21T11:13:50,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/b22e6aeccdfb4362b8ac7e7608c435d7 2024-11-21T11:13:50,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/0577bfc3aa434c42b0ecd0c5a4fe1cab as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/0577bfc3aa434c42b0ecd0c5a4fe1cab 2024-11-21T11:13:50,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/0577bfc3aa434c42b0ecd0c5a4fe1cab, entries=200, sequenceid=212, filesize=14.2 K 2024-11-21T11:13:50,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/c9ee4ca2c333450e91e8eb32ea874f14 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/c9ee4ca2c333450e91e8eb32ea874f14 2024-11-21T11:13:50,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/c9ee4ca2c333450e91e8eb32ea874f14, entries=150, sequenceid=212, filesize=11.9 K 2024-11-21T11:13:50,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/b22e6aeccdfb4362b8ac7e7608c435d7 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/b22e6aeccdfb4362b8ac7e7608c435d7 2024-11-21T11:13:50,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/b22e6aeccdfb4362b8ac7e7608c435d7, entries=150, sequenceid=212, filesize=11.9 K 2024-11-21T11:13:50,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for cb944dfe54fbd2ebb234cb2046a89860 in 521ms, sequenceid=212, compaction requested=true 2024-11-21T11:13:50,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:50,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb944dfe54fbd2ebb234cb2046a89860:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:13:50,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:13:50,380 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:13:50,381 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:13:50,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb944dfe54fbd2ebb234cb2046a89860:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:13:50,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:13:50,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb944dfe54fbd2ebb234cb2046a89860:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:13:50,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:13:50,383 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:13:50,383 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): cb944dfe54fbd2ebb234cb2046a89860/A is initiating minor compaction (all files) 2024-11-21T11:13:50,383 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:13:50,383 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb944dfe54fbd2ebb234cb2046a89860/A in TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:50,383 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): cb944dfe54fbd2ebb234cb2046a89860/B is initiating minor compaction (all files) 2024-11-21T11:13:50,383 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb944dfe54fbd2ebb234cb2046a89860/B in TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:50,383 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/b87f05ef8332415eabf861593aca33f6, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/2c0a563e77b2415d8a5ca25e5318b63d, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/0577bfc3aa434c42b0ecd0c5a4fe1cab] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp, totalSize=38.4 K 2024-11-21T11:13:50,383 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/9aebd59358764fd7bdf53eb86d1cecc1, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/9ef407c96a774fce8ee62a170525a518, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/c9ee4ca2c333450e91e8eb32ea874f14] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp, totalSize=36.0 K 2024-11-21T11:13:50,384 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting b87f05ef8332415eabf861593aca33f6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732187627323 2024-11-21T11:13:50,384 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 9aebd59358764fd7bdf53eb86d1cecc1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732187627323 2024-11-21T11:13:50,384 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c0a563e77b2415d8a5ca25e5318b63d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732187627999 2024-11-21T11:13:50,385 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ef407c96a774fce8ee62a170525a518, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732187627999 2024-11-21T11:13:50,385 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0577bfc3aa434c42b0ecd0c5a4fe1cab, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732187629200 2024-11-21T11:13:50,385 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting c9ee4ca2c333450e91e8eb32ea874f14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732187629200 2024-11-21T11:13:50,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T11:13:50,395 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:13:50,396 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:13:50,399 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb944dfe54fbd2ebb234cb2046a89860#A#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:13:50,399 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/f7fb6958f9104161ae0f46eeeff901da is 50, key is test_row_0/A:col10/1732187629857/Put/seqid=0 2024-11-21T11:13:50,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-21T11:13:50,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T11:13:50,404 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:13:50,406 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:13:50,406 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:13:50,409 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb944dfe54fbd2ebb234cb2046a89860#B#compaction#49 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:13:50,410 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/f2b51a4cc9954cffa3886e7f68f75f22 is 50, key is test_row_0/B:col10/1732187629857/Put/seqid=0 2024-11-21T11:13:50,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741885_1061 (size=12697) 2024-11-21T11:13:50,432 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/f7fb6958f9104161ae0f46eeeff901da as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/f7fb6958f9104161ae0f46eeeff901da 2024-11-21T11:13:50,440 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb944dfe54fbd2ebb234cb2046a89860/A of cb944dfe54fbd2ebb234cb2046a89860 into f7fb6958f9104161ae0f46eeeff901da(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:13:50,440 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:50,440 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., storeName=cb944dfe54fbd2ebb234cb2046a89860/A, priority=13, startTime=1732187630380; duration=0sec 2024-11-21T11:13:50,440 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:13:50,441 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb944dfe54fbd2ebb234cb2046a89860:A 2024-11-21T11:13:50,441 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:13:50,442 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:13:50,442 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): cb944dfe54fbd2ebb234cb2046a89860/C is initiating minor compaction (all files) 2024-11-21T11:13:50,443 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb944dfe54fbd2ebb234cb2046a89860/C in TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:50,443 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/5fff6e08bb644e7296422260088b9c7e, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/2b3ef9e752ca42f5b4acd393cabd67e6, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/b22e6aeccdfb4362b8ac7e7608c435d7] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp, totalSize=36.0 K 2024-11-21T11:13:50,443 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fff6e08bb644e7296422260088b9c7e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732187627323 2024-11-21T11:13:50,444 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b3ef9e752ca42f5b4acd393cabd67e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732187627999 2024-11-21T11:13:50,444 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting b22e6aeccdfb4362b8ac7e7608c435d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732187629200 2024-11-21T11:13:50,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741886_1062 (size=12697) 2024-11-21T11:13:50,457 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/f2b51a4cc9954cffa3886e7f68f75f22 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/f2b51a4cc9954cffa3886e7f68f75f22 2024-11-21T11:13:50,459 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb944dfe54fbd2ebb234cb2046a89860#C#compaction#50 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:13:50,460 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/9c509d32e97e48b6ba5ba3d8f0b3d77f is 50, key is test_row_0/C:col10/1732187629857/Put/seqid=0 2024-11-21T11:13:50,468 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb944dfe54fbd2ebb234cb2046a89860/B of cb944dfe54fbd2ebb234cb2046a89860 into f2b51a4cc9954cffa3886e7f68f75f22(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:13:50,469 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:50,469 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., storeName=cb944dfe54fbd2ebb234cb2046a89860/B, priority=13, startTime=1732187630380; duration=0sec 2024-11-21T11:13:50,469 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:13:50,469 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb944dfe54fbd2ebb234cb2046a89860:B 2024-11-21T11:13:50,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741887_1063 (size=12697) 2024-11-21T11:13:50,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T11:13:50,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:13:50,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb944dfe54fbd2ebb234cb2046a89860 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-21T11:13:50,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=A 2024-11-21T11:13:50,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:50,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=B 2024-11-21T11:13:50,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:50,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=C 2024-11-21T11:13:50,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:50,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/69cd450bf36c483a857e71334695e8d1 is 50, key is test_row_0/A:col10/1732187630527/Put/seqid=0 2024-11-21T11:13:50,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741888_1064 (size=16931) 2024-11-21T11:13:50,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/69cd450bf36c483a857e71334695e8d1 2024-11-21T11:13:50,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T11:13:50,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:50,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:50,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:50,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:50,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:50,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:50,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187690553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:50,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:50,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:50,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:50,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187690558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:50,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:50,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:50,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187690560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:50,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:50,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187690564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:50,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:50,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:50,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/38e26fd01c104d2385109622f0f8f168 is 50, key is test_row_0/B:col10/1732187630527/Put/seqid=0 2024-11-21T11:13:50,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741889_1065 (size=12151) 2024-11-21T11:13:50,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/38e26fd01c104d2385109622f0f8f168 2024-11-21T11:13:50,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/6d691ab7e5434ccaaa4dc353c3e8f454 is 50, key is test_row_0/C:col10/1732187630527/Put/seqid=0 2024-11-21T11:13:50,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741890_1066 (size=12151) 2024-11-21T11:13:50,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:50,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187690666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:50,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:50,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:50,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187690676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:50,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:50,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:50,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:50,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187690677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:50,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187690676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:50,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T11:13:50,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T11:13:50,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:50,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:50,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:50,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:50,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:50,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:50,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T11:13:50,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:50,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:50,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:50,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:50,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:50,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:50,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:50,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187690877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:50,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:50,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:50,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187690886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:50,887 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/9c509d32e97e48b6ba5ba3d8f0b3d77f as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/9c509d32e97e48b6ba5ba3d8f0b3d77f 2024-11-21T11:13:50,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:50,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187690886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:50,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:50,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:50,896 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb944dfe54fbd2ebb234cb2046a89860/C of cb944dfe54fbd2ebb234cb2046a89860 into 9c509d32e97e48b6ba5ba3d8f0b3d77f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:13:50,896 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:50,896 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., storeName=cb944dfe54fbd2ebb234cb2046a89860/C, priority=13, startTime=1732187630382; duration=0sec 2024-11-21T11:13:50,896 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:13:50,896 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb944dfe54fbd2ebb234cb2046a89860:C 2024-11-21T11:13:50,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:50,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187690889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:50,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:50,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T11:13:51,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T11:13:51,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:51,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:51,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:51,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:51,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:51,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:51,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/6d691ab7e5434ccaaa4dc353c3e8f454 2024-11-21T11:13:51,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/69cd450bf36c483a857e71334695e8d1 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/69cd450bf36c483a857e71334695e8d1 2024-11-21T11:13:51,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/69cd450bf36c483a857e71334695e8d1, entries=250, sequenceid=232, filesize=16.5 K 2024-11-21T11:13:51,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/38e26fd01c104d2385109622f0f8f168 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/38e26fd01c104d2385109622f0f8f168 2024-11-21T11:13:51,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/38e26fd01c104d2385109622f0f8f168, entries=150, sequenceid=232, filesize=11.9 K 2024-11-21T11:13:51,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/6d691ab7e5434ccaaa4dc353c3e8f454 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/6d691ab7e5434ccaaa4dc353c3e8f454 2024-11-21T11:13:51,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/6d691ab7e5434ccaaa4dc353c3e8f454, entries=150, sequenceid=232, filesize=11.9 K 2024-11-21T11:13:51,084 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for cb944dfe54fbd2ebb234cb2046a89860 in 554ms, sequenceid=232, compaction requested=false 2024-11-21T11:13:51,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:51,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T11:13:51,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:51,186 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing cb944dfe54fbd2ebb234cb2046a89860 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T11:13:51,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=A 2024-11-21T11:13:51,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:51,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=B 2024-11-21T11:13:51,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:51,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=C 2024-11-21T11:13:51,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:51,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:13:51,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:51,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/9a78c589fbd24b8b84b0f2698d48f863 is 50, key is test_row_0/A:col10/1732187630550/Put/seqid=0 2024-11-21T11:13:51,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741891_1067 (size=12151) 2024-11-21T11:13:51,201 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/9a78c589fbd24b8b84b0f2698d48f863 2024-11-21T11:13:51,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/b819590d7295498d877248ef8197f4e6 is 50, key is test_row_0/B:col10/1732187630550/Put/seqid=0 2024-11-21T11:13:51,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187691221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:51,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:51,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187691223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:51,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:51,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:51,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:51,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:51,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:51,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:51,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:51,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187691231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:51,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:51,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:51,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:51,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:51,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187691232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:51,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741892_1068 (size=12151) 2024-11-21T11:13:51,241 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/b819590d7295498d877248ef8197f4e6 2024-11-21T11:13:51,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/1739e4fceb28464bb2d2b935bf13a339 is 50, key is test_row_0/C:col10/1732187630550/Put/seqid=0 2024-11-21T11:13:51,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741893_1069 (size=12151) 2024-11-21T11:13:51,271 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/1739e4fceb28464bb2d2b935bf13a339 2024-11-21T11:13:51,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/9a78c589fbd24b8b84b0f2698d48f863 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/9a78c589fbd24b8b84b0f2698d48f863 2024-11-21T11:13:51,286 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/9a78c589fbd24b8b84b0f2698d48f863, entries=150, sequenceid=247, filesize=11.9 K 2024-11-21T11:13:51,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/b819590d7295498d877248ef8197f4e6 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/b819590d7295498d877248ef8197f4e6 2024-11-21T11:13:51,294 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/b819590d7295498d877248ef8197f4e6, entries=150, sequenceid=247, filesize=11.9 K 2024-11-21T11:13:51,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/1739e4fceb28464bb2d2b935bf13a339 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/1739e4fceb28464bb2d2b935bf13a339 2024-11-21T11:13:51,304 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/1739e4fceb28464bb2d2b935bf13a339, entries=150, sequenceid=247, filesize=11.9 K 2024-11-21T11:13:51,306 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for cb944dfe54fbd2ebb234cb2046a89860 in 120ms, sequenceid=247, compaction requested=true 2024-11-21T11:13:51,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:51,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:51,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-21T11:13:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-21T11:13:51,313 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-21T11:13:51,313 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 903 msec 2024-11-21T11:13:51,316 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 918 msec 2024-11-21T11:13:51,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:13:51,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb944dfe54fbd2ebb234cb2046a89860 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-21T11:13:51,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=A 2024-11-21T11:13:51,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:51,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=B 2024-11-21T11:13:51,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:51,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=C 2024-11-21T11:13:51,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:51,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/4f51187a49e049eb88a44095a3dc9aa4 is 50, key is test_row_0/A:col10/1732187631220/Put/seqid=0 2024-11-21T11:13:51,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741894_1070 (size=14741) 2024-11-21T11:13:51,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/4f51187a49e049eb88a44095a3dc9aa4 2024-11-21T11:13:51,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:51,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187691358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:51,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:51,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:51,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:51,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:51,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187691359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:51,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:51,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187691362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:51,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:51,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:51,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:51,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:51,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:51,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:51,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187691365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:51,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:51,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:51,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:51,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/c3714a86a64d42119e91af9d6e4f3b2f is 50, key is test_row_0/B:col10/1732187631220/Put/seqid=0 2024-11-21T11:13:51,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741895_1071 (size=12301) 2024-11-21T11:13:51,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187691475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:51,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:51,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:51,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:51,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187691476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:51,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:51,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187691476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:51,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:51,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:51,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:51,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:51,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:51,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187691476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:51,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:51,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:51,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T11:13:51,536 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:13:51,536 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:13:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-21T11:13:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T11:13:51,539 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:13:51,540 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:13:51,540 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:13:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T11:13:51,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:51,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187691685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187691685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:51,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:51,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:51,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:51,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:51,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:51,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:51,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:51,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187691685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187691685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:51,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:51,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:51,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:51,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:51,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:51,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:51,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-21T11:13:51,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:51,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:51,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:51,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:51,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:51,791 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/c3714a86a64d42119e91af9d6e4f3b2f 2024-11-21T11:13:51,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/d268d8687c4c42bc9e239f81e6113895 is 50, key is test_row_0/C:col10/1732187631220/Put/seqid=0 2024-11-21T11:13:51,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741896_1072 (size=12301) 2024-11-21T11:13:51,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/d268d8687c4c42bc9e239f81e6113895 2024-11-21T11:13:51,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-21T11:13:51,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:51,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:51,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:51,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:51,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:51,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/4f51187a49e049eb88a44095a3dc9aa4 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/4f51187a49e049eb88a44095a3dc9aa4 2024-11-21T11:13:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T11:13:51,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/4f51187a49e049eb88a44095a3dc9aa4, entries=200, sequenceid=267, filesize=14.4 K 2024-11-21T11:13:51,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/c3714a86a64d42119e91af9d6e4f3b2f as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/c3714a86a64d42119e91af9d6e4f3b2f 2024-11-21T11:13:51,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/c3714a86a64d42119e91af9d6e4f3b2f, entries=150, sequenceid=267, filesize=12.0 K 2024-11-21T11:13:51,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/d268d8687c4c42bc9e239f81e6113895 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/d268d8687c4c42bc9e239f81e6113895 2024-11-21T11:13:51,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/d268d8687c4c42bc9e239f81e6113895, entries=150, sequenceid=267, filesize=12.0 K 2024-11-21T11:13:51,886 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for cb944dfe54fbd2ebb234cb2046a89860 in 548ms, sequenceid=267, compaction requested=true 2024-11-21T11:13:51,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:51,886 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T11:13:51,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb944dfe54fbd2ebb234cb2046a89860:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:13:51,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:13:51,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb944dfe54fbd2ebb234cb2046a89860:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:13:51,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:13:51,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb944dfe54fbd2ebb234cb2046a89860:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:13:51,887 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T11:13:51,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:13:51,888 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56520 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T11:13:51,888 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): cb944dfe54fbd2ebb234cb2046a89860/A is initiating minor compaction (all files) 2024-11-21T11:13:51,889 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb944dfe54fbd2ebb234cb2046a89860/A in TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:51,889 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/f7fb6958f9104161ae0f46eeeff901da, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/69cd450bf36c483a857e71334695e8d1, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/9a78c589fbd24b8b84b0f2698d48f863, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/4f51187a49e049eb88a44095a3dc9aa4] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp, totalSize=55.2 K 2024-11-21T11:13:51,889 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49300 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T11:13:51,889 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): cb944dfe54fbd2ebb234cb2046a89860/B is initiating minor compaction (all files) 2024-11-21T11:13:51,889 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb944dfe54fbd2ebb234cb2046a89860/B in TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:51,890 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/f2b51a4cc9954cffa3886e7f68f75f22, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/38e26fd01c104d2385109622f0f8f168, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/b819590d7295498d877248ef8197f4e6, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/c3714a86a64d42119e91af9d6e4f3b2f] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp, totalSize=48.1 K 2024-11-21T11:13:51,890 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7fb6958f9104161ae0f46eeeff901da, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732187629200 2024-11-21T11:13:51,890 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting f2b51a4cc9954cffa3886e7f68f75f22, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732187629200 2024-11-21T11:13:51,891 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 38e26fd01c104d2385109622f0f8f168, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732187629889 2024-11-21T11:13:51,891 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69cd450bf36c483a857e71334695e8d1, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732187629885 2024-11-21T11:13:51,892 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting b819590d7295498d877248ef8197f4e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732187630542 2024-11-21T11:13:51,892 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a78c589fbd24b8b84b0f2698d48f863, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732187630542 2024-11-21T11:13:51,893 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting c3714a86a64d42119e91af9d6e4f3b2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732187631220 2024-11-21T11:13:51,893 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f51187a49e049eb88a44095a3dc9aa4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732187631214 2024-11-21T11:13:51,912 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb944dfe54fbd2ebb234cb2046a89860#A#compaction#61 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:13:51,913 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/77a195ce3ba0413086b0d9ec2e42d564 is 50, key is test_row_0/A:col10/1732187631220/Put/seqid=0 2024-11-21T11:13:51,918 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb944dfe54fbd2ebb234cb2046a89860#B#compaction#60 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:13:51,919 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/b56797ec72c74028a61394f819b95cf1 is 50, key is test_row_0/B:col10/1732187631220/Put/seqid=0 2024-11-21T11:13:51,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741897_1073 (size=12983) 2024-11-21T11:13:51,955 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/b56797ec72c74028a61394f819b95cf1 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/b56797ec72c74028a61394f819b95cf1 2024-11-21T11:13:51,967 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cb944dfe54fbd2ebb234cb2046a89860/B of cb944dfe54fbd2ebb234cb2046a89860 into b56797ec72c74028a61394f819b95cf1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:13:51,967 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:51,967 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., storeName=cb944dfe54fbd2ebb234cb2046a89860/B, priority=12, startTime=1732187631887; duration=0sec 2024-11-21T11:13:51,967 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:13:51,967 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb944dfe54fbd2ebb234cb2046a89860:B 2024-11-21T11:13:51,968 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T11:13:51,969 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49300 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T11:13:51,969 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): cb944dfe54fbd2ebb234cb2046a89860/C is initiating minor compaction (all files) 2024-11-21T11:13:51,969 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb944dfe54fbd2ebb234cb2046a89860/C in TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:51,970 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/9c509d32e97e48b6ba5ba3d8f0b3d77f, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/6d691ab7e5434ccaaa4dc353c3e8f454, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/1739e4fceb28464bb2d2b935bf13a339, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/d268d8687c4c42bc9e239f81e6113895] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp, totalSize=48.1 K 2024-11-21T11:13:51,970 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c509d32e97e48b6ba5ba3d8f0b3d77f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732187629200 2024-11-21T11:13:51,971 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d691ab7e5434ccaaa4dc353c3e8f454, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732187629889 2024-11-21T11:13:51,972 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 1739e4fceb28464bb2d2b935bf13a339, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732187630542 2024-11-21T11:13:51,972 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting d268d8687c4c42bc9e239f81e6113895, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732187631220 2024-11-21T11:13:51,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741898_1074 (size=12983) 2024-11-21T11:13:51,984 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/77a195ce3ba0413086b0d9ec2e42d564 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/77a195ce3ba0413086b0d9ec2e42d564 2024-11-21T11:13:51,998 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb944dfe54fbd2ebb234cb2046a89860#C#compaction#62 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:13:51,998 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cb944dfe54fbd2ebb234cb2046a89860/A of cb944dfe54fbd2ebb234cb2046a89860 into 77a195ce3ba0413086b0d9ec2e42d564(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:13:51,998 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:51,998 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., storeName=cb944dfe54fbd2ebb234cb2046a89860/A, priority=12, startTime=1732187631886; duration=0sec 2024-11-21T11:13:51,999 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:13:51,999 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb944dfe54fbd2ebb234cb2046a89860:A 2024-11-21T11:13:51,999 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/6603fbd31f7941c4b693301542ecde2d is 50, key is test_row_0/C:col10/1732187631220/Put/seqid=0 2024-11-21T11:13:52,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:13:52,001 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb944dfe54fbd2ebb234cb2046a89860 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-21T11:13:52,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=A 2024-11-21T11:13:52,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:52,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=B 2024-11-21T11:13:52,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:52,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=C 2024-11-21T11:13:52,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:52,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741899_1075 (size=12983) 2024-11-21T11:13:52,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-21T11:13:52,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:52,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:52,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:52,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:52,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:52,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:52,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/884fbb9482e4447f8c8e20b31b2c2c23 is 50, key is test_row_0/A:col10/1732187631361/Put/seqid=0 2024-11-21T11:13:52,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:52,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187692038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:52,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:52,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:52,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:52,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741900_1076 (size=12301) 2024-11-21T11:13:52,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:52,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187692042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:52,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:52,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:52,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:52,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:52,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187692044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:52,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:52,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187692044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:52,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:52,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/884fbb9482e4447f8c8e20b31b2c2c23 2024-11-21T11:13:52,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:52,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:52,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:52,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:52,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:52,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/cb145f814b6d437f89c05557b13e1a35 is 50, key is test_row_0/B:col10/1732187631361/Put/seqid=0 2024-11-21T11:13:52,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741901_1077 (size=12301) 2024-11-21T11:13:52,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:52,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187692156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:52,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:52,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:52,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:52,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:52,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187692156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:52,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-21T11:13:52,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:52,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:52,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:52,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:52,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:52,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:52,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:52,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:52,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T11:13:52,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:52,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:52,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187692166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:52,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:52,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187692166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:52,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:52,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:52,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:52,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:52,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:52,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:52,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40396 deadline: 1732187692215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:52,217 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_1' in cb944dfe54fbd2ebb234cb2046a89860 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:13:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:52,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-21T11:13:52,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:52,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:52,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:52,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:52,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:52,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:52,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:52,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187692366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:52,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:52,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:52,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:52,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187692375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:52,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:52,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:52,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:52,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187692375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:52,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187692376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:52,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:52,414 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/6603fbd31f7941c4b693301542ecde2d as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/6603fbd31f7941c4b693301542ecde2d 2024-11-21T11:13:52,426 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cb944dfe54fbd2ebb234cb2046a89860/C of cb944dfe54fbd2ebb234cb2046a89860 into 6603fbd31f7941c4b693301542ecde2d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:13:52,426 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:52,426 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., storeName=cb944dfe54fbd2ebb234cb2046a89860/C, priority=12, startTime=1732187631887; duration=0sec 2024-11-21T11:13:52,426 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:13:52,426 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb944dfe54fbd2ebb234cb2046a89860:C 2024-11-21T11:13:52,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-21T11:13:52,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:52,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:52,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:52,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:52,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:52,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/cb145f814b6d437f89c05557b13e1a35 2024-11-21T11:13:52,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/cdfa1315312144ffa3f8978109fcd3e0 is 50, key is test_row_0/C:col10/1732187631361/Put/seqid=0 2024-11-21T11:13:52,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741902_1078 (size=12301) 2024-11-21T11:13:52,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-21T11:13:52,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:52,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:52,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:52,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:52,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:52,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T11:13:52,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:52,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187692676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:52,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:52,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:52,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:52,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187692685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:52,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:52,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:52,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:52,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:52,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187692686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:52,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:52,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187692686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:52,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:52,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:52,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:52,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-21T11:13:52,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:52,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:52,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:52,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:52,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:52,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:52,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/cdfa1315312144ffa3f8978109fcd3e0 2024-11-21T11:13:52,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/884fbb9482e4447f8c8e20b31b2c2c23 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/884fbb9482e4447f8c8e20b31b2c2c23 2024-11-21T11:13:52,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-21T11:13:52,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:52,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:52,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:52,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:52,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:52,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/884fbb9482e4447f8c8e20b31b2c2c23, entries=150, sequenceid=283, filesize=12.0 K 2024-11-21T11:13:52,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:52,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/cb145f814b6d437f89c05557b13e1a35 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/cb145f814b6d437f89c05557b13e1a35 2024-11-21T11:13:52,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/cb145f814b6d437f89c05557b13e1a35, entries=150, sequenceid=283, filesize=12.0 K 2024-11-21T11:13:52,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/cdfa1315312144ffa3f8978109fcd3e0 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/cdfa1315312144ffa3f8978109fcd3e0 2024-11-21T11:13:52,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/cdfa1315312144ffa3f8978109fcd3e0, entries=150, sequenceid=283, filesize=12.0 K 2024-11-21T11:13:52,960 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for cb944dfe54fbd2ebb234cb2046a89860 in 959ms, sequenceid=283, compaction requested=false 2024-11-21T11:13:52,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:53,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-21T11:13:53,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:53,095 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing cb944dfe54fbd2ebb234cb2046a89860 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T11:13:53,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=A 2024-11-21T11:13:53,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:53,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=B 2024-11-21T11:13:53,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:53,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=C 2024-11-21T11:13:53,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:53,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/955a194c146c449a805dc74e2248f7bb is 50, key is test_row_0/A:col10/1732187632040/Put/seqid=0 2024-11-21T11:13:53,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741903_1079 (size=12301) 2024-11-21T11:13:53,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:13:53,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:53,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:53,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187693203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:53,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:53,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:53,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:53,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:53,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:53,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187693208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:53,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187693207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:53,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:53,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:53,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:53,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:53,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:53,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187693209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:53,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:53,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:53,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:53,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:53,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:53,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:53,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187693315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:53,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:53,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:53,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:53,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:53,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187693325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:53,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:53,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:53,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:53,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:53,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187693326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:53,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:53,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:53,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:53,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:53,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187693326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:53,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:53,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:53,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:53,506 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/955a194c146c449a805dc74e2248f7bb 2024-11-21T11:13:53,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/8f32d7a81df94d20a13bd09a516d4252 is 50, key is test_row_0/B:col10/1732187632040/Put/seqid=0 2024-11-21T11:13:53,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741904_1080 (size=12301) 2024-11-21T11:13:53,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:53,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187693526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:53,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:53,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:53,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:53,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:53,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187693535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:53,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:53,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:53,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:53,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:53,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187693535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:53,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:53,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:53,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:53,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:53,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187693535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:53,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:53,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:53,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:53,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T11:13:53,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:53,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187693835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:53,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:53,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:53,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:53,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:53,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187693845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:53,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:53,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:53,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:53,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:53,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187693845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:53,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:53,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:53,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:53,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:53,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187693846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:53,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:53,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:53,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:53,923 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/8f32d7a81df94d20a13bd09a516d4252 2024-11-21T11:13:53,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/5363c2a43534485aab187292ae44bf2e is 50, key is test_row_0/C:col10/1732187632040/Put/seqid=0 2024-11-21T11:13:53,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741905_1081 (size=12301) 2024-11-21T11:13:53,941 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/5363c2a43534485aab187292ae44bf2e 2024-11-21T11:13:53,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/955a194c146c449a805dc74e2248f7bb as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/955a194c146c449a805dc74e2248f7bb 2024-11-21T11:13:53,954 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/955a194c146c449a805dc74e2248f7bb, entries=150, sequenceid=303, filesize=12.0 K 2024-11-21T11:13:53,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/8f32d7a81df94d20a13bd09a516d4252 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/8f32d7a81df94d20a13bd09a516d4252 2024-11-21T11:13:53,961 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/8f32d7a81df94d20a13bd09a516d4252, entries=150, sequenceid=303, filesize=12.0 K 2024-11-21T11:13:53,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/5363c2a43534485aab187292ae44bf2e as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/5363c2a43534485aab187292ae44bf2e 2024-11-21T11:13:53,968 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/5363c2a43534485aab187292ae44bf2e, entries=150, sequenceid=303, filesize=12.0 K 2024-11-21T11:13:53,969 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for cb944dfe54fbd2ebb234cb2046a89860 in 874ms, sequenceid=303, compaction requested=true 2024-11-21T11:13:53,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:53,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:53,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-21T11:13:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-21T11:13:53,973 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-21T11:13:53,973 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4310 sec 2024-11-21T11:13:53,975 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.4380 sec 2024-11-21T11:13:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:13:54,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb944dfe54fbd2ebb234cb2046a89860 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T11:13:54,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=A 2024-11-21T11:13:54,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:54,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=B 2024-11-21T11:13:54,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:54,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=C 2024-11-21T11:13:54,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:54,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/a78c8db040e74af180217dc07cb18c0b is 50, key is test_row_0/A:col10/1732187633206/Put/seqid=0 2024-11-21T11:13:54,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741906_1082 (size=14741) 2024-11-21T11:13:54,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/a78c8db040e74af180217dc07cb18c0b 2024-11-21T11:13:54,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/d5b6a76b7bd34cda81cf17d92455e828 is 50, key is test_row_0/B:col10/1732187633206/Put/seqid=0 2024-11-21T11:13:54,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:54,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187694379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:54,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:54,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:54,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:54,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741907_1083 (size=12301) 2024-11-21T11:13:54,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:54,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187694381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:54,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:54,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187694382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:54,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:54,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:54,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:54,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:54,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187694384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:54,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:54,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:54,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:54,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:54,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:54,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:54,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:54,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187694495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:54,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:54,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:54,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:54,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187694495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:54,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:54,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:54,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:54,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187694495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:54,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:54,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187694496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:54,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:54,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:54,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:54,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:54,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:54,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:54,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:54,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187694706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:54,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:54,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:54,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:54,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:54,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187694706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:54,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:54,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:54,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:54,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:54,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187694706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:54,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:54,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:54,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:54,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187694706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:54,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:54,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:54,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:54,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/d5b6a76b7bd34cda81cf17d92455e828 2024-11-21T11:13:54,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/3ec8fa6ed83e417f9185f9824bf31904 is 50, key is test_row_0/C:col10/1732187633206/Put/seqid=0 2024-11-21T11:13:54,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741908_1084 (size=12301) 2024-11-21T11:13:55,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187695015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:55,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:55,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187695016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187695016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:55,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187695016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:55,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:55,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:55,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:55,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/3ec8fa6ed83e417f9185f9824bf31904 2024-11-21T11:13:55,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/a78c8db040e74af180217dc07cb18c0b as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/a78c8db040e74af180217dc07cb18c0b 2024-11-21T11:13:55,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/a78c8db040e74af180217dc07cb18c0b, entries=200, sequenceid=317, filesize=14.4 K 2024-11-21T11:13:55,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/d5b6a76b7bd34cda81cf17d92455e828 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/d5b6a76b7bd34cda81cf17d92455e828 2024-11-21T11:13:55,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/d5b6a76b7bd34cda81cf17d92455e828, entries=150, sequenceid=317, filesize=12.0 K 2024-11-21T11:13:55,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/3ec8fa6ed83e417f9185f9824bf31904 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/3ec8fa6ed83e417f9185f9824bf31904 2024-11-21T11:13:55,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/3ec8fa6ed83e417f9185f9824bf31904, entries=150, sequenceid=317, filesize=12.0 K 2024-11-21T11:13:55,242 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for cb944dfe54fbd2ebb234cb2046a89860 in 895ms, sequenceid=317, compaction requested=true 2024-11-21T11:13:55,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:55,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb944dfe54fbd2ebb234cb2046a89860:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:13:55,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:13:55,242 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T11:13:55,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb944dfe54fbd2ebb234cb2046a89860:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:13:55,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:13:55,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb944dfe54fbd2ebb234cb2046a89860:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:13:55,242 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T11:13:55,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:13:55,244 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T11:13:55,244 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52326 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T11:13:55,244 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): cb944dfe54fbd2ebb234cb2046a89860/A is initiating minor compaction (all files) 2024-11-21T11:13:55,244 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): cb944dfe54fbd2ebb234cb2046a89860/B is initiating minor compaction (all files) 2024-11-21T11:13:55,244 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb944dfe54fbd2ebb234cb2046a89860/A in TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:55,244 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb944dfe54fbd2ebb234cb2046a89860/B in TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:55,244 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/b56797ec72c74028a61394f819b95cf1, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/cb145f814b6d437f89c05557b13e1a35, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/8f32d7a81df94d20a13bd09a516d4252, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/d5b6a76b7bd34cda81cf17d92455e828] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp, totalSize=48.7 K 2024-11-21T11:13:55,244 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/77a195ce3ba0413086b0d9ec2e42d564, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/884fbb9482e4447f8c8e20b31b2c2c23, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/955a194c146c449a805dc74e2248f7bb, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/a78c8db040e74af180217dc07cb18c0b] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp, totalSize=51.1 K 2024-11-21T11:13:55,245 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting b56797ec72c74028a61394f819b95cf1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732187631220 2024-11-21T11:13:55,245 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77a195ce3ba0413086b0d9ec2e42d564, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732187631220 2024-11-21T11:13:55,245 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting cb145f814b6d437f89c05557b13e1a35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732187631361 2024-11-21T11:13:55,245 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 884fbb9482e4447f8c8e20b31b2c2c23, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732187631361 2024-11-21T11:13:55,246 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f32d7a81df94d20a13bd09a516d4252, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732187632033 2024-11-21T11:13:55,246 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 955a194c146c449a805dc74e2248f7bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732187632033 2024-11-21T11:13:55,246 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting a78c8db040e74af180217dc07cb18c0b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732187633202 2024-11-21T11:13:55,246 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting d5b6a76b7bd34cda81cf17d92455e828, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732187633206 2024-11-21T11:13:55,260 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb944dfe54fbd2ebb234cb2046a89860#A#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:13:55,261 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/717020b39a5f48688e9253a44a61d3fc is 50, key is test_row_0/A:col10/1732187633206/Put/seqid=0 2024-11-21T11:13:55,263 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb944dfe54fbd2ebb234cb2046a89860#B#compaction#73 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:13:55,264 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/6416d4282af34200b62c0024e979d104 is 50, key is test_row_0/B:col10/1732187633206/Put/seqid=0 2024-11-21T11:13:55,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741909_1085 (size=13119) 2024-11-21T11:13:55,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741910_1086 (size=13119) 2024-11-21T11:13:55,285 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/6416d4282af34200b62c0024e979d104 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/6416d4282af34200b62c0024e979d104 2024-11-21T11:13:55,296 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cb944dfe54fbd2ebb234cb2046a89860/B of cb944dfe54fbd2ebb234cb2046a89860 into 6416d4282af34200b62c0024e979d104(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:13:55,296 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:55,296 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., storeName=cb944dfe54fbd2ebb234cb2046a89860/B, priority=12, startTime=1732187635242; duration=0sec 2024-11-21T11:13:55,299 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:13:55,299 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb944dfe54fbd2ebb234cb2046a89860:B 2024-11-21T11:13:55,299 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T11:13:55,301 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T11:13:55,301 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): cb944dfe54fbd2ebb234cb2046a89860/C is initiating minor compaction (all files) 2024-11-21T11:13:55,301 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb944dfe54fbd2ebb234cb2046a89860/C in TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:55,301 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/6603fbd31f7941c4b693301542ecde2d, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/cdfa1315312144ffa3f8978109fcd3e0, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/5363c2a43534485aab187292ae44bf2e, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/3ec8fa6ed83e417f9185f9824bf31904] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp, totalSize=48.7 K 2024-11-21T11:13:55,302 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 6603fbd31f7941c4b693301542ecde2d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732187631220 2024-11-21T11:13:55,302 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting cdfa1315312144ffa3f8978109fcd3e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732187631361 2024-11-21T11:13:55,303 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 5363c2a43534485aab187292ae44bf2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732187632033 2024-11-21T11:13:55,303 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ec8fa6ed83e417f9185f9824bf31904, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732187633206 2024-11-21T11:13:55,319 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb944dfe54fbd2ebb234cb2046a89860#C#compaction#74 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:13:55,320 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/ce6076e5486248f5ace8469cf262075f is 50, key is test_row_0/C:col10/1732187633206/Put/seqid=0 2024-11-21T11:13:55,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741911_1087 (size=13119) 2024-11-21T11:13:55,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:13:55,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb944dfe54fbd2ebb234cb2046a89860 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T11:13:55,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=A 2024-11-21T11:13:55,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:55,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=B 2024-11-21T11:13:55,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:55,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=C 2024-11-21T11:13:55,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:55,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/e21df74d702d4c2ba18d1e2b8e2e4cec is 50, key is test_row_0/A:col10/1732187635527/Put/seqid=0 2024-11-21T11:13:55,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187695547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:55,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:55,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:55,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:55,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187695551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:55,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:55,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:55,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:55,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:55,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741912_1088 (size=17181) 2024-11-21T11:13:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187695553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:55,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:55,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:55,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:55,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187695554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:55,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:55,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:55,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:55,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187695666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:55,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:55,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:55,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187695666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:55,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:55,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:55,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187695666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:55,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:55,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:55,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:55,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:55,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:55,677 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/717020b39a5f48688e9253a44a61d3fc as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/717020b39a5f48688e9253a44a61d3fc 2024-11-21T11:13:55,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187695675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:55,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:55,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:55,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:55,685 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cb944dfe54fbd2ebb234cb2046a89860/A of cb944dfe54fbd2ebb234cb2046a89860 into 717020b39a5f48688e9253a44a61d3fc(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:13:55,685 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:55,685 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., storeName=cb944dfe54fbd2ebb234cb2046a89860/A, priority=12, startTime=1732187635242; duration=0sec 2024-11-21T11:13:55,686 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:13:55,687 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb944dfe54fbd2ebb234cb2046a89860:A 2024-11-21T11:13:55,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T11:13:55,706 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:13:55,707 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:13:55,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-21T11:13:55,710 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:13:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T11:13:55,711 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:13:55,711 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:13:55,741 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/ce6076e5486248f5ace8469cf262075f as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/ce6076e5486248f5ace8469cf262075f 2024-11-21T11:13:55,749 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cb944dfe54fbd2ebb234cb2046a89860/C of cb944dfe54fbd2ebb234cb2046a89860 into ce6076e5486248f5ace8469cf262075f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:13:55,749 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:55,749 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., storeName=cb944dfe54fbd2ebb234cb2046a89860/C, priority=12, startTime=1732187635242; duration=0sec 2024-11-21T11:13:55,750 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:13:55,750 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb944dfe54fbd2ebb234cb2046a89860:C 2024-11-21T11:13:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T11:13:55,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-21T11:13:55,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:55,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:55,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:55,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:55,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:55,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187695876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:55,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:55,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:55,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:55,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187695876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:55,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:55,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:55,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:55,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:55,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187695876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:55,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:55,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:55,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:55,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187695886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:55,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:55,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:55,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:55,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/e21df74d702d4c2ba18d1e2b8e2e4cec 2024-11-21T11:13:55,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/5a0ef51a678545b8820c8776911d4214 is 50, key is test_row_0/B:col10/1732187635527/Put/seqid=0 2024-11-21T11:13:55,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741913_1089 (size=12301) 2024-11-21T11:13:56,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-21T11:13:56,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:56,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:56,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:56,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:56,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T11:13:56,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-21T11:13:56,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:56,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:56,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:56,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:56,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:56,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:56,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:56,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187696195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:56,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:56,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187696195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:56,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:56,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:56,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:56,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:56,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187696195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:56,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:56,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:56,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:56,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:56,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:56,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:56,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:56,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187696196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:56,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:56,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:56,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:56,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-21T11:13:56,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:56,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:56,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:56,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:56,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:56,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T11:13:56,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:56,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/5a0ef51a678545b8820c8776911d4214 2024-11-21T11:13:56,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/f259879c61d24d47a386d7d25602f397 is 50, key is test_row_0/C:col10/1732187635527/Put/seqid=0 2024-11-21T11:13:56,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741914_1090 (size=12301) 2024-11-21T11:13:56,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-21T11:13:56,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:56,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:56,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:56,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:56,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:56,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:56,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-21T11:13:56,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:56,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:56,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:56,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:56,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:56,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:56,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:56,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187696705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:56,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:56,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:56,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:56,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:56,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187696705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:56,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:56,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:56,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:56,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:56,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:56,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187696716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:56,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187696716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:56,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:56,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:56,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:56,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:56,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:56,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:56,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-21T11:13:56,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:56,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:56,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:56,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:56,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:56,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:13:56,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T11:13:56,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/f259879c61d24d47a386d7d25602f397 2024-11-21T11:13:56,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/e21df74d702d4c2ba18d1e2b8e2e4cec as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/e21df74d702d4c2ba18d1e2b8e2e4cec 2024-11-21T11:13:56,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/e21df74d702d4c2ba18d1e2b8e2e4cec, entries=250, sequenceid=338, filesize=16.8 K 2024-11-21T11:13:56,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/5a0ef51a678545b8820c8776911d4214 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/5a0ef51a678545b8820c8776911d4214 2024-11-21T11:13:56,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/5a0ef51a678545b8820c8776911d4214, entries=150, sequenceid=338, filesize=12.0 K 2024-11-21T11:13:56,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/f259879c61d24d47a386d7d25602f397 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/f259879c61d24d47a386d7d25602f397 2024-11-21T11:13:56,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/f259879c61d24d47a386d7d25602f397, entries=150, sequenceid=338, filesize=12.0 K 2024-11-21T11:13:56,888 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for cb944dfe54fbd2ebb234cb2046a89860 in 1358ms, sequenceid=338, compaction requested=false 2024-11-21T11:13:56,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:56,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-21T11:13:56,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:56,951 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing cb944dfe54fbd2ebb234cb2046a89860 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-21T11:13:56,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=A 2024-11-21T11:13:56,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:56,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=B 2024-11-21T11:13:56,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:56,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=C 2024-11-21T11:13:56,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:56,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/3d47f3dd2fbf4d96944ecf7b2a746764 is 50, key is test_row_0/A:col10/1732187635550/Put/seqid=0 2024-11-21T11:13:56,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741915_1091 (size=12301) 2024-11-21T11:13:56,972 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/3d47f3dd2fbf4d96944ecf7b2a746764 2024-11-21T11:13:56,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/71132b2ddcd44ab5ba2c95459eda94bd is 50, key is test_row_0/B:col10/1732187635550/Put/seqid=0 2024-11-21T11:13:57,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741916_1092 (size=12301) 2024-11-21T11:13:57,416 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/71132b2ddcd44ab5ba2c95459eda94bd 2024-11-21T11:13:57,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/7745c46c9c6148c9964f8dc107c2f43b is 50, key is test_row_0/C:col10/1732187635550/Put/seqid=0 2024-11-21T11:13:57,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741917_1093 (size=12301) 2024-11-21T11:13:57,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:13:57,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:13:57,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:57,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187697751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:57,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:57,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187697752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:57,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:57,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:57,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:57,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:57,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187697752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:57,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:57,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:57,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:57,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:57,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:57,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:57,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:57,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187697756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:57,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:57,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:57,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:57,835 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/7745c46c9c6148c9964f8dc107c2f43b 2024-11-21T11:13:57,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/3d47f3dd2fbf4d96944ecf7b2a746764 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/3d47f3dd2fbf4d96944ecf7b2a746764 2024-11-21T11:13:57,852 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/3d47f3dd2fbf4d96944ecf7b2a746764, entries=150, sequenceid=353, filesize=12.0 K 2024-11-21T11:13:57,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/71132b2ddcd44ab5ba2c95459eda94bd as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/71132b2ddcd44ab5ba2c95459eda94bd 2024-11-21T11:13:57,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T11:13:57,861 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/71132b2ddcd44ab5ba2c95459eda94bd, entries=150, sequenceid=353, filesize=12.0 K 2024-11-21T11:13:57,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/7745c46c9c6148c9964f8dc107c2f43b as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/7745c46c9c6148c9964f8dc107c2f43b 2024-11-21T11:13:57,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:57,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187697866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:57,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:57,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:57,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187697866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:57,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187697866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:57,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:57,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187697866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:57,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:57,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:57,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:57,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:57,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:57,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:57,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:57,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:57,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:57,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:57,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:57,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:57,872 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/7745c46c9c6148c9964f8dc107c2f43b, entries=150, sequenceid=353, filesize=12.0 K 2024-11-21T11:13:57,877 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for cb944dfe54fbd2ebb234cb2046a89860 in 926ms, sequenceid=353, compaction requested=true 2024-11-21T11:13:57,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:57,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:57,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-21T11:13:57,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-21T11:13:57,882 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-21T11:13:57,882 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1690 sec 2024-11-21T11:13:57,885 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.1760 sec 2024-11-21T11:13:58,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:13:58,081 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb944dfe54fbd2ebb234cb2046a89860 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-21T11:13:58,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732187637733, max timestamp=1732187638081Num uniques 859; 2024-11-21T11:13:58,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732187637733, max timestamp=1732187638081Num uniques 859; 2024-11-21T11:13:58,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=A 2024-11-21T11:13:58,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T11:13:58,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T11:13:58,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:13:58,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=B 2024-11-21T11:13:58,092 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732187637750, max timestamp=1732187638081Num uniques 859; 2024-11-21T11:13:58,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:13:58,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=C 2024-11-21T11:13:58,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:13:58,093 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-21T11:13:58,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/41399c8215fe48229be9f8050db616a2 is 50, key is test_row_0/A:col10/1732187638079/Put/seqid=0 2024-11-21T11:13:58,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741918_1094 (size=12301) 2024-11-21T11:13:58,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/41399c8215fe48229be9f8050db616a2 2024-11-21T11:13:58,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/8fbb834a22da43d488c1bbb40dfe3de5 is 50, key is test_row_0/B:col10/1732187638079/Put/seqid=0 2024-11-21T11:13:58,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732187638121, max timestamp=1732187638145Num uniques 859; 2024-11-21T11:13:58,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T11:13:58,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732187638121, max timestamp=1732187638146Num uniques 859; 2024-11-21T11:13:58,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T11:13:58,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732187638121, max timestamp=1732187638145Num uniques 859; 2024-11-21T11:13:58,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T11:13:58,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741919_1095 (size=12301) 2024-11-21T11:13:58,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/8fbb834a22da43d488c1bbb40dfe3de5 2024-11-21T11:13:58,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/ca6ceacd8825467eb6ff229b019b83fc is 50, key is test_row_0/C:col10/1732187638079/Put/seqid=0 2024-11-21T11:13:58,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741920_1096 (size=12301) 2024-11-21T11:13:58,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732187638182, max timestamp=1732187638206Num uniques 1118; 2024-11-21T11:13:58,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-11-21T11:13:58,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732187638182, max timestamp=1732187638206Num uniques 1018; 2024-11-21T11:13:58,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-21T11:13:58,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732187638182, max timestamp=1732187638206Num uniques 1018; 2024-11-21T11:13:58,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-21T11:13:58,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:58,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187698249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:58,257 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=427, cellSize=19552, totalHeapSize=44648, min timestamp=1732187638223, max timestamp=1732187638251Num uniques 1127; 2024-11-21T11:13:58,257 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=19552, suffix heap size=145176,new segment heap size=44648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=427 2024-11-21T11:13:58,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:58,258 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=427, cellSize=19552, totalHeapSize=44648, min timestamp=1732187638223, max timestamp=1732187638251Num uniques 1127; 2024-11-21T11:13:58,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:58,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:58,258 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=19552, suffix heap size=145176,new segment heap size=44648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=427 2024-11-21T11:13:58,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=427, cellSize=19552, totalHeapSize=44648, min timestamp=1732187638223, max timestamp=1732187638251Num uniques 1127; 2024-11-21T11:13:58,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=19552, suffix heap size=145176,new segment heap size=44648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=427 2024-11-21T11:13:58,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:58,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187698308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:58,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:58,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:58,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:58,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:58,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187698314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:58,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:58,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187698316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:58,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:58,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:58,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:58,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:58,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:58,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:58,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:58,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187698365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:58,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:58,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:58,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:58,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:58,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187698430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:58,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:58,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:58,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187698432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:58,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:58,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:58,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:58,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187698433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:58,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:58,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:58,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:58,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:58,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:58,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:58,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:58,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187698576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:58,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:58,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:58,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:58,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.55 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/ca6ceacd8825467eb6ff229b019b83fc 2024-11-21T11:13:58,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/41399c8215fe48229be9f8050db616a2 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/41399c8215fe48229be9f8050db616a2 2024-11-21T11:13:58,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/41399c8215fe48229be9f8050db616a2, entries=150, sequenceid=374, filesize=12.0 K 2024-11-21T11:13:58,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/8fbb834a22da43d488c1bbb40dfe3de5 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/8fbb834a22da43d488c1bbb40dfe3de5 2024-11-21T11:13:58,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/8fbb834a22da43d488c1bbb40dfe3de5, entries=150, sequenceid=374, filesize=12.0 K 2024-11-21T11:13:58,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/ca6ceacd8825467eb6ff229b019b83fc as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/ca6ceacd8825467eb6ff229b019b83fc 2024-11-21T11:13:58,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/ca6ceacd8825467eb6ff229b019b83fc, entries=150, sequenceid=374, filesize=12.0 K 2024-11-21T11:13:58,644 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~30.91 KB/31655, heapSize ~75.43 KB/77240, currentSize=167.72 KB/171750 for cb944dfe54fbd2ebb234cb2046a89860 in 563ms, sequenceid=374, compaction requested=true 2024-11-21T11:13:58,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:58,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb944dfe54fbd2ebb234cb2046a89860:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:13:58,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:13:58,645 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T11:13:58,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb944dfe54fbd2ebb234cb2046a89860:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:13:58,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:13:58,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb944dfe54fbd2ebb234cb2046a89860:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:13:58,645 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T11:13:58,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:13:58,647 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54902 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T11:13:58,648 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): cb944dfe54fbd2ebb234cb2046a89860/A is initiating minor compaction (all files) 2024-11-21T11:13:58,648 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb944dfe54fbd2ebb234cb2046a89860/A in TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:58,648 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/717020b39a5f48688e9253a44a61d3fc, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/e21df74d702d4c2ba18d1e2b8e2e4cec, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/3d47f3dd2fbf4d96944ecf7b2a746764, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/41399c8215fe48229be9f8050db616a2] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp, totalSize=53.6 K 2024-11-21T11:13:58,648 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T11:13:58,648 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): cb944dfe54fbd2ebb234cb2046a89860/B is initiating minor compaction (all files) 2024-11-21T11:13:58,648 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 717020b39a5f48688e9253a44a61d3fc, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732187633206 2024-11-21T11:13:58,648 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb944dfe54fbd2ebb234cb2046a89860/B in TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:58,649 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/6416d4282af34200b62c0024e979d104, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/5a0ef51a678545b8820c8776911d4214, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/71132b2ddcd44ab5ba2c95459eda94bd, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/8fbb834a22da43d488c1bbb40dfe3de5] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp, totalSize=48.8 K 2024-11-21T11:13:58,649 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 6416d4282af34200b62c0024e979d104, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732187633206 2024-11-21T11:13:58,649 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting e21df74d702d4c2ba18d1e2b8e2e4cec, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732187634377 2024-11-21T11:13:58,650 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a0ef51a678545b8820c8776911d4214, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732187634382 2024-11-21T11:13:58,650 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d47f3dd2fbf4d96944ecf7b2a746764, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1732187635543 2024-11-21T11:13:58,650 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41399c8215fe48229be9f8050db616a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732187637750 2024-11-21T11:13:58,650 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 71132b2ddcd44ab5ba2c95459eda94bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1732187635543 2024-11-21T11:13:58,651 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fbb834a22da43d488c1bbb40dfe3de5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732187637750 2024-11-21T11:13:58,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:13:58,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb944dfe54fbd2ebb234cb2046a89860 3/3 column families, dataSize=174.43 KB heapSize=480.09 KB 2024-11-21T11:13:58,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=A 2024-11-21T11:13:58,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-21T11:13:58,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=B 2024-11-21T11:13:58,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-21T11:13:58,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=C 2024-11-21T11:13:58,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-21T11:13:58,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:58,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187698654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:58,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:58,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:58,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:58,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:58,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187698657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:58,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:58,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:58,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:58,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:58,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187698658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:58,678 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb944dfe54fbd2ebb234cb2046a89860#B#compaction#84 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:13:58,679 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/011902cf4b8147d0b32e01882f87b0a2 is 50, key is test_row_0/B:col10/1732187638079/Put/seqid=0 2024-11-21T11:13:58,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/ef3997ca224a4909b318c99da0017041 is 50, key is test_row_0/A:col10/1732187638312/Put/seqid=0 2024-11-21T11:13:58,686 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb944dfe54fbd2ebb234cb2046a89860#A#compaction#86 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:13:58,687 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/17d53ce17deb4328a39e47d659a398c1 is 50, key is test_row_0/A:col10/1732187638079/Put/seqid=0 2024-11-21T11:13:58,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741921_1097 (size=17181) 2024-11-21T11:13:58,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741922_1098 (size=13255) 2024-11-21T11:13:58,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741923_1099 (size=13255) 2024-11-21T11:13:58,735 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/17d53ce17deb4328a39e47d659a398c1 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/17d53ce17deb4328a39e47d659a398c1 2024-11-21T11:13:58,746 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cb944dfe54fbd2ebb234cb2046a89860/A of cb944dfe54fbd2ebb234cb2046a89860 into 17d53ce17deb4328a39e47d659a398c1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:13:58,746 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:58,746 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., storeName=cb944dfe54fbd2ebb234cb2046a89860/A, priority=12, startTime=1732187638645; duration=0sec 2024-11-21T11:13:58,746 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:13:58,746 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb944dfe54fbd2ebb234cb2046a89860:A 2024-11-21T11:13:58,746 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T11:13:58,749 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T11:13:58,750 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): cb944dfe54fbd2ebb234cb2046a89860/C is initiating minor compaction (all files) 2024-11-21T11:13:58,750 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb944dfe54fbd2ebb234cb2046a89860/C in TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:13:58,752 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/ce6076e5486248f5ace8469cf262075f, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/f259879c61d24d47a386d7d25602f397, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/7745c46c9c6148c9964f8dc107c2f43b, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/ca6ceacd8825467eb6ff229b019b83fc] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp, totalSize=48.8 K 2024-11-21T11:13:58,752 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce6076e5486248f5ace8469cf262075f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732187633206 2024-11-21T11:13:58,753 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting f259879c61d24d47a386d7d25602f397, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732187634382 2024-11-21T11:13:58,754 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7745c46c9c6148c9964f8dc107c2f43b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1732187635543 2024-11-21T11:13:58,754 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca6ceacd8825467eb6ff229b019b83fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732187637750 2024-11-21T11:13:58,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:58,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187698765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:58,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:58,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187698766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:58,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:58,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:58,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:58,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:58,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:58,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:58,771 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb944dfe54fbd2ebb234cb2046a89860#C#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:13:58,772 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/d25f4caa26ec4b0ebf05c0c933e8f909 is 50, key is test_row_0/C:col10/1732187638079/Put/seqid=0 2024-11-21T11:13:58,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741924_1100 (size=13255) 2024-11-21T11:13:58,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:58,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187698776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:58,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:58,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:58,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:58,800 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/d25f4caa26ec4b0ebf05c0c933e8f909 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/d25f4caa26ec4b0ebf05c0c933e8f909 2024-11-21T11:13:58,810 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cb944dfe54fbd2ebb234cb2046a89860/C of cb944dfe54fbd2ebb234cb2046a89860 into d25f4caa26ec4b0ebf05c0c933e8f909(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:13:58,810 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:58,810 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., storeName=cb944dfe54fbd2ebb234cb2046a89860/C, priority=12, startTime=1732187638645; duration=0sec 2024-11-21T11:13:58,810 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:13:58,810 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb944dfe54fbd2ebb234cb2046a89860:C 2024-11-21T11:13:58,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:58,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187698895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:58,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:58,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:58,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:58,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:58,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187698976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:58,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:58,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:58,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:58,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:58,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187698976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:58,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:58,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:58,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:59,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:59,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187698995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:59,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:59,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:59,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:59,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/ef3997ca224a4909b318c99da0017041 2024-11-21T11:13:59,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/794fa9e4c6464353b4cd53159eada40e is 50, key is test_row_0/B:col10/1732187638312/Put/seqid=0 2024-11-21T11:13:59,113 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/011902cf4b8147d0b32e01882f87b0a2 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/011902cf4b8147d0b32e01882f87b0a2 2024-11-21T11:13:59,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741925_1101 (size=12301) 2024-11-21T11:13:59,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/794fa9e4c6464353b4cd53159eada40e 2024-11-21T11:13:59,123 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cb944dfe54fbd2ebb234cb2046a89860/B of cb944dfe54fbd2ebb234cb2046a89860 into 011902cf4b8147d0b32e01882f87b0a2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:13:59,123 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:59,123 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., storeName=cb944dfe54fbd2ebb234cb2046a89860/B, priority=12, startTime=1732187638645; duration=0sec 2024-11-21T11:13:59,124 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:13:59,124 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb944dfe54fbd2ebb234cb2046a89860:B 2024-11-21T11:13:59,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/e8b41f87614248c38399347f10876639 is 50, key is test_row_0/C:col10/1732187638312/Put/seqid=0 2024-11-21T11:13:59,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741926_1102 (size=12301) 2024-11-21T11:13:59,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:59,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187699285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:59,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:59,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:59,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:59,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:59,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187699285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:59,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:59,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:59,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:59,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:59,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187699316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:59,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:59,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:59,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:59,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:59,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187699406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:59,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:59,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:59,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:59,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/e8b41f87614248c38399347f10876639 2024-11-21T11:13:59,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/ef3997ca224a4909b318c99da0017041 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/ef3997ca224a4909b318c99da0017041 2024-11-21T11:13:59,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/ef3997ca224a4909b318c99da0017041, entries=250, sequenceid=446, filesize=16.8 K 2024-11-21T11:13:59,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/794fa9e4c6464353b4cd53159eada40e as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/794fa9e4c6464353b4cd53159eada40e 2024-11-21T11:13:59,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/794fa9e4c6464353b4cd53159eada40e, entries=150, sequenceid=446, filesize=12.0 K 2024-11-21T11:13:59,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/e8b41f87614248c38399347f10876639 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/e8b41f87614248c38399347f10876639 2024-11-21T11:13:59,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/e8b41f87614248c38399347f10876639, entries=150, sequenceid=446, filesize=12.0 K 2024-11-21T11:13:59,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~480.05 KB/491568, currentSize=13.42 KB/13740 for cb944dfe54fbd2ebb234cb2046a89860 in 916ms, sequenceid=446, compaction requested=false 2024-11-21T11:13:59,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:13:59,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:13:59,804 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb944dfe54fbd2ebb234cb2046a89860 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T11:13:59,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=A 2024-11-21T11:13:59,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:59,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=B 2024-11-21T11:13:59,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:59,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=C 2024-11-21T11:13:59,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:13:59,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/703e0ae9a9404606944a6f87d64ed322 is 50, key is test_row_0/A:col10/1732187639799/Put/seqid=0 2024-11-21T11:13:59,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741927_1103 (size=12301) 2024-11-21T11:13:59,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/703e0ae9a9404606944a6f87d64ed322 2024-11-21T11:13:59,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/2dd95528c3be4d77a540fa906f9de38f is 50, key is test_row_0/B:col10/1732187639799/Put/seqid=0 2024-11-21T11:13:59,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:59,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187699854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:59,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:59,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:59,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:59,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T11:13:59,865 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:13:59,866 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:13:59,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-21T11:13:59,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:59,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187699862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:59,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:59,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187699862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:59,869 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:13:59,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:59,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:59,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:59,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:59,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:59,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:59,870 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:13:59,870 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:13:59,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T11:13:59,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741928_1104 (size=12301) 2024-11-21T11:13:59,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/2dd95528c3be4d77a540fa906f9de38f 2024-11-21T11:13:59,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/b0e4992f32614a9784554b1fd62a68a9 is 50, key is test_row_0/C:col10/1732187639799/Put/seqid=0 2024-11-21T11:13:59,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741929_1105 (size=12301) 2024-11-21T11:13:59,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:59,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187699965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:59,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:59,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:59,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:59,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T11:13:59,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:59,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187699975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:59,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:59,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:59,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:13:59,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:13:59,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187699976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:13:59,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:59,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:13:59,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:00,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T11:14:00,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:14:00,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:14:00,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:14:00,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:00,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:00,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:00,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T11:14:00,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:00,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:14:00,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:14:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187700175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:00,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:14:00,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:00,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:00,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T11:14:00,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:00,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187700186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:00,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:00,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:00,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:00,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:00,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187700186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:00,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:00,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:00,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:00,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/b0e4992f32614a9784554b1fd62a68a9 2024-11-21T11:14:00,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/703e0ae9a9404606944a6f87d64ed322 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/703e0ae9a9404606944a6f87d64ed322 2024-11-21T11:14:00,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/703e0ae9a9404606944a6f87d64ed322, entries=150, sequenceid=459, filesize=12.0 K 2024-11-21T11:14:00,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/2dd95528c3be4d77a540fa906f9de38f as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/2dd95528c3be4d77a540fa906f9de38f 2024-11-21T11:14:00,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/2dd95528c3be4d77a540fa906f9de38f, entries=150, sequenceid=459, filesize=12.0 K 2024-11-21T11:14:00,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/b0e4992f32614a9784554b1fd62a68a9 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/b0e4992f32614a9784554b1fd62a68a9 2024-11-21T11:14:00,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/b0e4992f32614a9784554b1fd62a68a9, entries=150, sequenceid=459, filesize=12.0 K 2024-11-21T11:14:00,322 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=127.47 KB/130530 for cb944dfe54fbd2ebb234cb2046a89860 in 518ms, sequenceid=459, compaction requested=true 2024-11-21T11:14:00,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:14:00,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb944dfe54fbd2ebb234cb2046a89860:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:14:00,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:00,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb944dfe54fbd2ebb234cb2046a89860:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:14:00,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:00,322 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:14:00,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb944dfe54fbd2ebb234cb2046a89860:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:14:00,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:14:00,322 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:14:00,323 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:14:00,324 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): cb944dfe54fbd2ebb234cb2046a89860/B is initiating minor compaction (all files) 2024-11-21T11:14:00,324 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb944dfe54fbd2ebb234cb2046a89860/B in TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:14:00,324 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/011902cf4b8147d0b32e01882f87b0a2, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/794fa9e4c6464353b4cd53159eada40e, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/2dd95528c3be4d77a540fa906f9de38f] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp, totalSize=37.0 K 2024-11-21T11:14:00,324 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42737 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:14:00,324 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): cb944dfe54fbd2ebb234cb2046a89860/A is initiating minor compaction (all files) 2024-11-21T11:14:00,324 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb944dfe54fbd2ebb234cb2046a89860/A in TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:14:00,324 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/17d53ce17deb4328a39e47d659a398c1, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/ef3997ca224a4909b318c99da0017041, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/703e0ae9a9404606944a6f87d64ed322] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp, totalSize=41.7 K 2024-11-21T11:14:00,325 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17d53ce17deb4328a39e47d659a398c1, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732187637750 2024-11-21T11:14:00,325 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 011902cf4b8147d0b32e01882f87b0a2, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732187637750 2024-11-21T11:14:00,326 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef3997ca224a4909b318c99da0017041, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1732187638303 2024-11-21T11:14:00,326 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 794fa9e4c6464353b4cd53159eada40e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1732187638303 2024-11-21T11:14:00,326 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 703e0ae9a9404606944a6f87d64ed322, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1732187639799 2024-11-21T11:14:00,327 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dd95528c3be4d77a540fa906f9de38f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1732187639799 2024-11-21T11:14:00,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T11:14:00,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:14:00,332 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing cb944dfe54fbd2ebb234cb2046a89860 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-11-21T11:14:00,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=A 2024-11-21T11:14:00,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:14:00,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=B 2024-11-21T11:14:00,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:14:00,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=C 2024-11-21T11:14:00,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:14:00,339 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb944dfe54fbd2ebb234cb2046a89860#A#compaction#93 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:00,340 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/2ec7b4fb1e304101b72e9a46cb997208 is 50, key is test_row_0/A:col10/1732187639799/Put/seqid=0 2024-11-21T11:14:00,340 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb944dfe54fbd2ebb234cb2046a89860#B#compaction#94 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:00,341 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/c1c83f5787744c7b80892f09ad968ea6 is 50, key is test_row_0/B:col10/1732187639799/Put/seqid=0 2024-11-21T11:14:00,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/d0200161e7c849e8829086db185579ef is 50, key is test_row_0/A:col10/1732187639849/Put/seqid=0 2024-11-21T11:14:00,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741930_1106 (size=13357) 2024-11-21T11:14:00,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741932_1108 (size=13357) 2024-11-21T11:14:00,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741931_1107 (size=12301) 2024-11-21T11:14:00,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:14:00,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:14:00,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:00,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187700462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:00,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:00,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:00,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:00,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:00,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187700485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:00,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:00,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:00,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:00,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T11:14:00,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:00,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187700496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:00,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:00,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:00,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:00,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:00,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187700496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:00,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:00,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:00,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:00,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:00,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187700565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:00,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:00,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:00,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:00,761 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/c1c83f5787744c7b80892f09ad968ea6 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/c1c83f5787744c7b80892f09ad968ea6 2024-11-21T11:14:00,767 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb944dfe54fbd2ebb234cb2046a89860/B of cb944dfe54fbd2ebb234cb2046a89860 into c1c83f5787744c7b80892f09ad968ea6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:00,767 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:14:00,767 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., storeName=cb944dfe54fbd2ebb234cb2046a89860/B, priority=13, startTime=1732187640322; duration=0sec 2024-11-21T11:14:00,767 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:14:00,768 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb944dfe54fbd2ebb234cb2046a89860:B 2024-11-21T11:14:00,768 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:14:00,771 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:14:00,771 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): cb944dfe54fbd2ebb234cb2046a89860/C is initiating minor compaction (all files) 2024-11-21T11:14:00,771 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb944dfe54fbd2ebb234cb2046a89860/C in TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:14:00,771 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/d25f4caa26ec4b0ebf05c0c933e8f909, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/e8b41f87614248c38399347f10876639, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/b0e4992f32614a9784554b1fd62a68a9] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp, totalSize=37.0 K 2024-11-21T11:14:00,772 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting d25f4caa26ec4b0ebf05c0c933e8f909, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732187637750 2024-11-21T11:14:00,773 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting e8b41f87614248c38399347f10876639, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1732187638303 2024-11-21T11:14:00,774 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting b0e4992f32614a9784554b1fd62a68a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1732187639799 2024-11-21T11:14:00,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:00,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187700775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:00,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:00,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:00,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:00,779 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/2ec7b4fb1e304101b72e9a46cb997208 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/2ec7b4fb1e304101b72e9a46cb997208 2024-11-21T11:14:00,779 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/d0200161e7c849e8829086db185579ef 2024-11-21T11:14:00,788 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb944dfe54fbd2ebb234cb2046a89860/A of cb944dfe54fbd2ebb234cb2046a89860 into 2ec7b4fb1e304101b72e9a46cb997208(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:00,789 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:14:00,789 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., storeName=cb944dfe54fbd2ebb234cb2046a89860/A, priority=13, startTime=1732187640322; duration=0sec 2024-11-21T11:14:00,789 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:00,789 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb944dfe54fbd2ebb234cb2046a89860:A 2024-11-21T11:14:00,795 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb944dfe54fbd2ebb234cb2046a89860#C#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:00,795 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/a441eba26d824b39a0cedd669290d0c2 is 50, key is test_row_0/C:col10/1732187639799/Put/seqid=0 2024-11-21T11:14:00,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/c3af42da5f9d4bc0b7b89233114673fc is 50, key is test_row_0/B:col10/1732187639849/Put/seqid=0 2024-11-21T11:14:00,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741933_1109 (size=13357) 2024-11-21T11:14:00,814 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/a441eba26d824b39a0cedd669290d0c2 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/a441eba26d824b39a0cedd669290d0c2 2024-11-21T11:14:00,821 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb944dfe54fbd2ebb234cb2046a89860/C of cb944dfe54fbd2ebb234cb2046a89860 into a441eba26d824b39a0cedd669290d0c2(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:00,821 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:14:00,821 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., storeName=cb944dfe54fbd2ebb234cb2046a89860/C, priority=13, startTime=1732187640322; duration=0sec 2024-11-21T11:14:00,821 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:00,821 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb944dfe54fbd2ebb234cb2046a89860:C 2024-11-21T11:14:00,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741934_1110 (size=12301) 2024-11-21T11:14:00,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:00,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187700996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:00,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:00,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:00,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:01,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T11:14:01,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:01,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187701005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:01,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:01,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:01,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:01,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:01,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187701005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:01,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:01,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:01,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:01,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:01,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187701086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:01,233 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/c3af42da5f9d4bc0b7b89233114673fc 2024-11-21T11:14:01,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/d80d93c68e634a30bffb5c774d85fbca is 50, key is test_row_0/C:col10/1732187639849/Put/seqid=0 2024-11-21T11:14:01,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741935_1111 (size=12301) 2024-11-21T11:14:01,248 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/d80d93c68e634a30bffb5c774d85fbca 2024-11-21T11:14:01,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/d0200161e7c849e8829086db185579ef as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/d0200161e7c849e8829086db185579ef 2024-11-21T11:14:01,259 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/d0200161e7c849e8829086db185579ef, entries=150, sequenceid=481, filesize=12.0 K 2024-11-21T11:14:01,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/c3af42da5f9d4bc0b7b89233114673fc as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/c3af42da5f9d4bc0b7b89233114673fc 2024-11-21T11:14:01,265 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/c3af42da5f9d4bc0b7b89233114673fc, entries=150, sequenceid=481, filesize=12.0 K 2024-11-21T11:14:01,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/d80d93c68e634a30bffb5c774d85fbca as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/d80d93c68e634a30bffb5c774d85fbca 2024-11-21T11:14:01,272 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/d80d93c68e634a30bffb5c774d85fbca, entries=150, sequenceid=481, filesize=12.0 K 2024-11-21T11:14:01,273 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for cb944dfe54fbd2ebb234cb2046a89860 in 941ms, sequenceid=481, compaction requested=false 2024-11-21T11:14:01,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:14:01,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:14:01,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-21T11:14:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-21T11:14:01,278 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-21T11:14:01,278 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4050 sec 2024-11-21T11:14:01,280 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.4130 sec 2024-11-21T11:14:01,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:14:01,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb944dfe54fbd2ebb234cb2046a89860 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-21T11:14:01,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=A 2024-11-21T11:14:01,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:01,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=B 2024-11-21T11:14:01,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:01,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=C 2024-11-21T11:14:01,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:01,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/35578da38be24e1481747c4542923200 is 50, key is test_row_0/A:col10/1732187641606/Put/seqid=0 2024-11-21T11:14:01,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741936_1112 (size=12301) 2024-11-21T11:14:01,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732187641678, max timestamp=1732187641718Num uniques 859; 2024-11-21T11:14:01,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732187641678, max timestamp=1732187641718Num uniques 859; 2024-11-21T11:14:01,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732187641678, max timestamp=1732187641718Num uniques 859; 2024-11-21T11:14:01,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T11:14:01,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T11:14:01,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T11:14:01,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:01,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187701817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:01,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:01,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:01,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:01,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:01,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187701925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:01,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:01,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:01,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:02,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:02,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40440 deadline: 1732187702006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:02,013 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_2' in cb944dfe54fbd2ebb234cb2046a89860 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:14:02,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:02,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:02,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:02,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T11:14:02,016 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:14:02,016 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:14:02,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-21T11:14:02,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:02,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40434 deadline: 1732187702015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:02,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T11:14:02,023 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:14:02,024 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:14:02,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:14:02,025 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_2' in cb944dfe54fbd2ebb234cb2046a89860 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:14:02,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:02,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:02,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:02,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/35578da38be24e1481747c4542923200 2024-11-21T11:14:02,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:02,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40456 deadline: 1732187702028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:02,034 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_1' in cb944dfe54fbd2ebb234cb2046a89860 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:14:02,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:02,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:02,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:02,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/1770377b91f242378c42b329399e0e91 is 50, key is test_row_0/B:col10/1732187641606/Put/seqid=0 2024-11-21T11:14:02,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741937_1113 (size=12301) 2024-11-21T11:14:02,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/1770377b91f242378c42b329399e0e91 2024-11-21T11:14:02,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/98ea7612dd464a5787d046f806d7a720 is 50, key is test_row_0/C:col10/1732187641606/Put/seqid=0 2024-11-21T11:14:02,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741938_1114 (size=12301) 2024-11-21T11:14:02,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T11:14:02,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:02,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187702136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:02,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:02,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:02,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:02,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T11:14:02,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:14:02,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:14:02,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:14:02,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:02,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:02,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:02,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:02,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40396 deadline: 1732187702275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:02,281 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_1' in cb944dfe54fbd2ebb234cb2046a89860 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18302 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:14:02,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:02,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:02,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:02,293 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-21T11:14:02,293 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:14:02,295 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-21T11:14:02,295 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:14:02,298 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-21T11:14:02,298 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:14:02,299 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:02,299 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:02,299 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:02,300 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:02,300 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:02,300 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:14:02,300 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:14:02,300 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:02,300 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:14:02,304 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-21T11:14:02,304 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:14:02,304 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:02,304 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:02,304 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:14:02,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T11:14:02,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:14:02,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:14:02,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:14:02,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:02,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:02,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:02,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T11:14:02,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:02,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40380 deadline: 1732187702445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:02,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:02,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:02,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:02,473 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/98ea7612dd464a5787d046f806d7a720 2024-11-21T11:14:02,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/35578da38be24e1481747c4542923200 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/35578da38be24e1481747c4542923200 2024-11-21T11:14:02,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/35578da38be24e1481747c4542923200, entries=150, sequenceid=495, filesize=12.0 K 2024-11-21T11:14:02,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/1770377b91f242378c42b329399e0e91 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/1770377b91f242378c42b329399e0e91 2024-11-21T11:14:02,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T11:14:02,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:14:02,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:14:02,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:14:02,486 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:02,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:02,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:02,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/1770377b91f242378c42b329399e0e91, entries=150, sequenceid=495, filesize=12.0 K 2024-11-21T11:14:02,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/98ea7612dd464a5787d046f806d7a720 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/98ea7612dd464a5787d046f806d7a720 2024-11-21T11:14:02,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/98ea7612dd464a5787d046f806d7a720, entries=150, sequenceid=495, filesize=12.0 K 2024-11-21T11:14:02,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for cb944dfe54fbd2ebb234cb2046a89860 in 888ms, sequenceid=495, compaction requested=true 2024-11-21T11:14:02,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:14:02,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb944dfe54fbd2ebb234cb2046a89860:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:14:02,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:02,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb944dfe54fbd2ebb234cb2046a89860:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:14:02,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:02,495 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:14:02,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb944dfe54fbd2ebb234cb2046a89860:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:14:02,495 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:14:02,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:14:02,496 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:14:02,496 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:14:02,496 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): cb944dfe54fbd2ebb234cb2046a89860/A is initiating minor compaction (all files) 2024-11-21T11:14:02,496 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): cb944dfe54fbd2ebb234cb2046a89860/B is initiating minor compaction (all files) 2024-11-21T11:14:02,496 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb944dfe54fbd2ebb234cb2046a89860/A in TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:14:02,496 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb944dfe54fbd2ebb234cb2046a89860/B in TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:14:02,496 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/2ec7b4fb1e304101b72e9a46cb997208, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/d0200161e7c849e8829086db185579ef, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/35578da38be24e1481747c4542923200] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp, totalSize=37.1 K 2024-11-21T11:14:02,496 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/c1c83f5787744c7b80892f09ad968ea6, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/c3af42da5f9d4bc0b7b89233114673fc, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/1770377b91f242378c42b329399e0e91] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp, totalSize=37.1 K 2024-11-21T11:14:02,497 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ec7b4fb1e304101b72e9a46cb997208, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1732187639799 2024-11-21T11:14:02,497 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting c1c83f5787744c7b80892f09ad968ea6, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1732187639799 2024-11-21T11:14:02,497 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0200161e7c849e8829086db185579ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732187639849 2024-11-21T11:14:02,497 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting c3af42da5f9d4bc0b7b89233114673fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732187639849 2024-11-21T11:14:02,498 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35578da38be24e1481747c4542923200, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=495, earliestPutTs=1732187640453 2024-11-21T11:14:02,498 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 1770377b91f242378c42b329399e0e91, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=495, earliestPutTs=1732187640453 2024-11-21T11:14:02,505 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb944dfe54fbd2ebb234cb2046a89860#A#compaction#102 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:02,505 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb944dfe54fbd2ebb234cb2046a89860#B#compaction#103 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:02,506 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/8f168225310b46618a40cbdf5d263e8c is 50, key is test_row_0/A:col10/1732187641606/Put/seqid=0 2024-11-21T11:14:02,506 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/ce00f5b7fdc445adafd121aef3c8f016 is 50, key is test_row_0/B:col10/1732187641606/Put/seqid=0 2024-11-21T11:14:02,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741940_1116 (size=13459) 2024-11-21T11:14:02,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741939_1115 (size=13459) 2024-11-21T11:14:02,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T11:14:02,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:14:02,640 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing cb944dfe54fbd2ebb234cb2046a89860 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-21T11:14:02,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=A 2024-11-21T11:14:02,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:14:02,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=B 2024-11-21T11:14:02,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:14:02,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=C 2024-11-21T11:14:02,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:14:02,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/f8215202c81e405d965c147b59cca651 is 50, key is test_row_0/A:col10/1732187641815/Put/seqid=0 2024-11-21T11:14:02,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T11:14:02,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741941_1117 (size=12301) 2024-11-21T11:14:02,917 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/ce00f5b7fdc445adafd121aef3c8f016 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/ce00f5b7fdc445adafd121aef3c8f016 2024-11-21T11:14:02,920 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/8f168225310b46618a40cbdf5d263e8c as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/8f168225310b46618a40cbdf5d263e8c 2024-11-21T11:14:02,924 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb944dfe54fbd2ebb234cb2046a89860/B of cb944dfe54fbd2ebb234cb2046a89860 into ce00f5b7fdc445adafd121aef3c8f016(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:02,924 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:14:02,924 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., storeName=cb944dfe54fbd2ebb234cb2046a89860/B, priority=13, startTime=1732187642495; duration=0sec 2024-11-21T11:14:02,924 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:14:02,924 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb944dfe54fbd2ebb234cb2046a89860:B 2024-11-21T11:14:02,924 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:14:02,926 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:14:02,926 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): cb944dfe54fbd2ebb234cb2046a89860/C is initiating minor compaction (all files) 2024-11-21T11:14:02,926 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb944dfe54fbd2ebb234cb2046a89860/C in TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:14:02,926 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/a441eba26d824b39a0cedd669290d0c2, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/d80d93c68e634a30bffb5c774d85fbca, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/98ea7612dd464a5787d046f806d7a720] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp, totalSize=37.1 K 2024-11-21T11:14:02,927 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb944dfe54fbd2ebb234cb2046a89860/A of cb944dfe54fbd2ebb234cb2046a89860 into 8f168225310b46618a40cbdf5d263e8c(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:02,927 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:14:02,927 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting a441eba26d824b39a0cedd669290d0c2, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1732187639799 2024-11-21T11:14:02,927 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., storeName=cb944dfe54fbd2ebb234cb2046a89860/A, priority=13, startTime=1732187642495; duration=0sec 2024-11-21T11:14:02,927 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:02,927 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb944dfe54fbd2ebb234cb2046a89860:A 2024-11-21T11:14:02,927 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting d80d93c68e634a30bffb5c774d85fbca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732187639849 2024-11-21T11:14:02,927 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 98ea7612dd464a5787d046f806d7a720, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=495, earliestPutTs=1732187640453 2024-11-21T11:14:02,946 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb944dfe54fbd2ebb234cb2046a89860#C#compaction#105 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:02,946 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/3b5efa73d6114ee496eab4b26f2ccc2d is 50, key is test_row_0/C:col10/1732187641606/Put/seqid=0 2024-11-21T11:14:02,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. as already flushing 2024-11-21T11:14:02,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:14:02,957 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-21T11:14:02,957 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:14:02,957 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:02,957 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:02,957 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:14:02,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741942_1118 (size=13459) 2024-11-21T11:14:02,965 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/3b5efa73d6114ee496eab4b26f2ccc2d as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/3b5efa73d6114ee496eab4b26f2ccc2d 2024-11-21T11:14:02,971 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb944dfe54fbd2ebb234cb2046a89860/C of cb944dfe54fbd2ebb234cb2046a89860 into 3b5efa73d6114ee496eab4b26f2ccc2d(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:02,971 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:14:02,971 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., storeName=cb944dfe54fbd2ebb234cb2046a89860/C, priority=13, startTime=1732187642495; duration=0sec 2024-11-21T11:14:02,971 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:02,971 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb944dfe54fbd2ebb234cb2046a89860:C 2024-11-21T11:14:03,049 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=530 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/f8215202c81e405d965c147b59cca651 2024-11-21T11:14:03,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/370f53aaa0d94eb889d3beb80e60afe4 is 50, key is test_row_0/B:col10/1732187641815/Put/seqid=0 2024-11-21T11:14:03,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741943_1119 (size=12301) 2024-11-21T11:14:03,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T11:14:03,463 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=530 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/370f53aaa0d94eb889d3beb80e60afe4 2024-11-21T11:14:03,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/996eecb2e6c7448792917a48b66c4d65 is 50, key is test_row_0/C:col10/1732187641815/Put/seqid=0 2024-11-21T11:14:03,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741944_1120 (size=12301) 2024-11-21T11:14:03,883 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=530 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/996eecb2e6c7448792917a48b66c4d65 2024-11-21T11:14:03,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/f8215202c81e405d965c147b59cca651 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/f8215202c81e405d965c147b59cca651 2024-11-21T11:14:03,893 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/f8215202c81e405d965c147b59cca651, entries=150, sequenceid=530, filesize=12.0 K 2024-11-21T11:14:03,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/370f53aaa0d94eb889d3beb80e60afe4 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/370f53aaa0d94eb889d3beb80e60afe4 2024-11-21T11:14:03,898 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/370f53aaa0d94eb889d3beb80e60afe4, entries=150, sequenceid=530, filesize=12.0 K 2024-11-21T11:14:03,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/996eecb2e6c7448792917a48b66c4d65 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/996eecb2e6c7448792917a48b66c4d65 2024-11-21T11:14:03,903 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/996eecb2e6c7448792917a48b66c4d65, entries=150, sequenceid=530, filesize=12.0 K 2024-11-21T11:14:03,904 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=6.71 KB/6870 for cb944dfe54fbd2ebb234cb2046a89860 in 1265ms, sequenceid=530, compaction requested=false 2024-11-21T11:14:03,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for cb944dfe54fbd2ebb234cb2046a89860: 2024-11-21T11:14:03,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:14:03,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-21T11:14:03,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-21T11:14:03,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-21T11:14:03,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8820 sec 2024-11-21T11:14:03,910 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.8930 sec 2024-11-21T11:14:04,027 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-21T11:14:04,028 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:14:04,028 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:04,028 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:04,028 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:14:04,037 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-21T11:14:04,037 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:14:04,037 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:04,037 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:04,037 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:14:04,057 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-21T11:14:04,057 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:14:04,057 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:04,057 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:04,057 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:14:04,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T11:14:04,166 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:14:06,655 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T11:14:12,377 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-21T11:14:12,377 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:14:12,377 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:12,377 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:12,377 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:14:12,378 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T11:14:12,378 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-11-21T11:14:12,378 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 125 2024-11-21T11:14:12,378 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-21T11:14:12,378 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-11-21T11:14:12,378 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-21T11:14:12,378 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T11:14:12,378 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2677 2024-11-21T11:14:12,378 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2563 2024-11-21T11:14:12,378 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T11:14:12,378 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1150 2024-11-21T11:14:12,378 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3447 rows 2024-11-21T11:14:12,378 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1142 2024-11-21T11:14:12,378 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3426 rows 2024-11-21T11:14:12,378 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T11:14:12,378 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T11:14:12,378 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:12,379 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:12,379 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:14:12,383 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T11:14:12,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T11:14:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T11:14:12,390 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732187652390"}]},"ts":"1732187652390"} 2024-11-21T11:14:12,392 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T11:14:12,392 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T11:14:12,394 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T11:14:12,397 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb944dfe54fbd2ebb234cb2046a89860, UNASSIGN}] 2024-11-21T11:14:12,398 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb944dfe54fbd2ebb234cb2046a89860, UNASSIGN 2024-11-21T11:14:12,399 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=cb944dfe54fbd2ebb234cb2046a89860, regionState=CLOSING, regionLocation=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:12,401 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb944dfe54fbd2ebb234cb2046a89860, UNASSIGN because future has completed 2024-11-21T11:14:12,402 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T11:14:12,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276}] 2024-11-21T11:14:12,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T11:14:12,559 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:14:12,559 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T11:14:12,559 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing cb944dfe54fbd2ebb234cb2046a89860, disabling compactions & flushes 2024-11-21T11:14:12,559 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:14:12,559 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:14:12,560 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. after waiting 0 ms 2024-11-21T11:14:12,560 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:14:12,560 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing cb944dfe54fbd2ebb234cb2046a89860 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-21T11:14:12,560 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=A 2024-11-21T11:14:12,560 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:12,560 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=B 2024-11-21T11:14:12,560 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:12,560 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cb944dfe54fbd2ebb234cb2046a89860, store=C 2024-11-21T11:14:12,560 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:12,565 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/49833122549944c397854cd5825c91ba is 50, key is test_row_0/A:col10/1732187642955/Put/seqid=0 2024-11-21T11:14:12,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741945_1121 (size=12301) 2024-11-21T11:14:12,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T11:14:12,970 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/49833122549944c397854cd5825c91ba 2024-11-21T11:14:12,978 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/d63cf3a8ba9d42f89cc3e20f87926044 is 50, key is test_row_0/B:col10/1732187642955/Put/seqid=0 2024-11-21T11:14:12,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741946_1122 (size=12301) 2024-11-21T11:14:13,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T11:14:13,383 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/d63cf3a8ba9d42f89cc3e20f87926044 2024-11-21T11:14:13,391 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/7fbaff4757294479ad61f4aeffee1828 is 50, key is test_row_0/C:col10/1732187642955/Put/seqid=0 2024-11-21T11:14:13,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741947_1123 (size=12301) 2024-11-21T11:14:13,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T11:14:13,796 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/7fbaff4757294479ad61f4aeffee1828 2024-11-21T11:14:13,801 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/A/49833122549944c397854cd5825c91ba as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/49833122549944c397854cd5825c91ba 2024-11-21T11:14:13,806 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/49833122549944c397854cd5825c91ba, entries=150, sequenceid=541, filesize=12.0 K 2024-11-21T11:14:13,806 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/B/d63cf3a8ba9d42f89cc3e20f87926044 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/d63cf3a8ba9d42f89cc3e20f87926044 2024-11-21T11:14:13,811 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/d63cf3a8ba9d42f89cc3e20f87926044, entries=150, sequenceid=541, filesize=12.0 K 2024-11-21T11:14:13,812 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/.tmp/C/7fbaff4757294479ad61f4aeffee1828 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/7fbaff4757294479ad61f4aeffee1828 2024-11-21T11:14:13,816 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/7fbaff4757294479ad61f4aeffee1828, entries=150, sequenceid=541, filesize=12.0 K 2024-11-21T11:14:13,817 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for cb944dfe54fbd2ebb234cb2046a89860 in 1256ms, sequenceid=541, compaction requested=true 2024-11-21T11:14:13,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/31ab7a47424249f39fbf9a067fafdf8f, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/4506b49795554c2e951f90a566b3beb8, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/7f273322d80649f6b901583db4d7b6e8, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/92c8255f646140969a73db28ffc9d04c, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/bfe8bca2c5cd49dba23961818ed5dab1, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/06d5796f4c0241a8b295e701ec446f86, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/c3b2f1d6b5fb425199acd349ecec76b2, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/15c6ac36db174b04bb3167f23d1618f1, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/15235ed32b464296a58efe255df2c0b7, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/c0617618cdc945c799b3aee8fcb81e16, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/5058748b6aab4498ac05d10dc4568d13, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/c0ec46da75be4713b194e727d7b864e3, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/60d23b4b7a3b498791770db943e040c0, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/b87f05ef8332415eabf861593aca33f6, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/2c0a563e77b2415d8a5ca25e5318b63d, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/0577bfc3aa434c42b0ecd0c5a4fe1cab, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/f7fb6958f9104161ae0f46eeeff901da, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/69cd450bf36c483a857e71334695e8d1, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/9a78c589fbd24b8b84b0f2698d48f863, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/4f51187a49e049eb88a44095a3dc9aa4, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/77a195ce3ba0413086b0d9ec2e42d564, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/884fbb9482e4447f8c8e20b31b2c2c23, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/955a194c146c449a805dc74e2248f7bb, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/a78c8db040e74af180217dc07cb18c0b, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/717020b39a5f48688e9253a44a61d3fc, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/e21df74d702d4c2ba18d1e2b8e2e4cec, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/3d47f3dd2fbf4d96944ecf7b2a746764, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/17d53ce17deb4328a39e47d659a398c1, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/41399c8215fe48229be9f8050db616a2, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/ef3997ca224a4909b318c99da0017041, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/2ec7b4fb1e304101b72e9a46cb997208, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/703e0ae9a9404606944a6f87d64ed322, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/d0200161e7c849e8829086db185579ef, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/35578da38be24e1481747c4542923200] to archive 2024-11-21T11:14:13,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T11:14:13,825 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/31ab7a47424249f39fbf9a067fafdf8f to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/31ab7a47424249f39fbf9a067fafdf8f 2024-11-21T11:14:13,827 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/4506b49795554c2e951f90a566b3beb8 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/4506b49795554c2e951f90a566b3beb8 2024-11-21T11:14:13,828 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/7f273322d80649f6b901583db4d7b6e8 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/7f273322d80649f6b901583db4d7b6e8 2024-11-21T11:14:13,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/92c8255f646140969a73db28ffc9d04c to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/92c8255f646140969a73db28ffc9d04c 2024-11-21T11:14:13,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/bfe8bca2c5cd49dba23961818ed5dab1 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/bfe8bca2c5cd49dba23961818ed5dab1 2024-11-21T11:14:13,832 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/06d5796f4c0241a8b295e701ec446f86 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/06d5796f4c0241a8b295e701ec446f86 2024-11-21T11:14:13,833 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/c3b2f1d6b5fb425199acd349ecec76b2 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/c3b2f1d6b5fb425199acd349ecec76b2 2024-11-21T11:14:13,834 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/15c6ac36db174b04bb3167f23d1618f1 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/15c6ac36db174b04bb3167f23d1618f1 2024-11-21T11:14:13,835 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/15235ed32b464296a58efe255df2c0b7 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/15235ed32b464296a58efe255df2c0b7 2024-11-21T11:14:13,837 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/c0617618cdc945c799b3aee8fcb81e16 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/c0617618cdc945c799b3aee8fcb81e16 2024-11-21T11:14:13,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/5058748b6aab4498ac05d10dc4568d13 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/5058748b6aab4498ac05d10dc4568d13 2024-11-21T11:14:13,839 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/c0ec46da75be4713b194e727d7b864e3 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/c0ec46da75be4713b194e727d7b864e3 2024-11-21T11:14:13,840 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/60d23b4b7a3b498791770db943e040c0 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/60d23b4b7a3b498791770db943e040c0 2024-11-21T11:14:13,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/b87f05ef8332415eabf861593aca33f6 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/b87f05ef8332415eabf861593aca33f6 2024-11-21T11:14:13,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/2c0a563e77b2415d8a5ca25e5318b63d to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/2c0a563e77b2415d8a5ca25e5318b63d 2024-11-21T11:14:13,844 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/0577bfc3aa434c42b0ecd0c5a4fe1cab to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/0577bfc3aa434c42b0ecd0c5a4fe1cab 2024-11-21T11:14:13,846 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/f7fb6958f9104161ae0f46eeeff901da to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/f7fb6958f9104161ae0f46eeeff901da 2024-11-21T11:14:13,847 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/69cd450bf36c483a857e71334695e8d1 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/69cd450bf36c483a857e71334695e8d1 2024-11-21T11:14:13,848 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/9a78c589fbd24b8b84b0f2698d48f863 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/9a78c589fbd24b8b84b0f2698d48f863 2024-11-21T11:14:13,849 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/4f51187a49e049eb88a44095a3dc9aa4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/4f51187a49e049eb88a44095a3dc9aa4 2024-11-21T11:14:13,851 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/77a195ce3ba0413086b0d9ec2e42d564 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/77a195ce3ba0413086b0d9ec2e42d564 2024-11-21T11:14:13,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/884fbb9482e4447f8c8e20b31b2c2c23 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/884fbb9482e4447f8c8e20b31b2c2c23 2024-11-21T11:14:13,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/955a194c146c449a805dc74e2248f7bb to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/955a194c146c449a805dc74e2248f7bb 2024-11-21T11:14:13,854 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/a78c8db040e74af180217dc07cb18c0b to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/a78c8db040e74af180217dc07cb18c0b 2024-11-21T11:14:13,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/717020b39a5f48688e9253a44a61d3fc to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/717020b39a5f48688e9253a44a61d3fc 2024-11-21T11:14:13,856 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/e21df74d702d4c2ba18d1e2b8e2e4cec to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/e21df74d702d4c2ba18d1e2b8e2e4cec 2024-11-21T11:14:13,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/3d47f3dd2fbf4d96944ecf7b2a746764 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/3d47f3dd2fbf4d96944ecf7b2a746764 2024-11-21T11:14:13,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/17d53ce17deb4328a39e47d659a398c1 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/17d53ce17deb4328a39e47d659a398c1 2024-11-21T11:14:13,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/41399c8215fe48229be9f8050db616a2 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/41399c8215fe48229be9f8050db616a2 2024-11-21T11:14:13,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/ef3997ca224a4909b318c99da0017041 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/ef3997ca224a4909b318c99da0017041 2024-11-21T11:14:13,862 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/2ec7b4fb1e304101b72e9a46cb997208 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/2ec7b4fb1e304101b72e9a46cb997208 2024-11-21T11:14:13,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/703e0ae9a9404606944a6f87d64ed322 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/703e0ae9a9404606944a6f87d64ed322 2024-11-21T11:14:13,864 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/d0200161e7c849e8829086db185579ef to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/d0200161e7c849e8829086db185579ef 2024-11-21T11:14:13,865 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/35578da38be24e1481747c4542923200 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/35578da38be24e1481747c4542923200 2024-11-21T11:14:13,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/8b6f85bef0074899bcb1d517efe1e818, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/fad3b6a8d0b74ab99c4cdf90aa668f83, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/bdf34b6149e84c2299fbb4e2d3873276, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/31d051962e5045d280c2d236a0d65c44, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/7ebaa14e3bdd4276b94cb9bd6f2f68ef, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/7eaec5076bf84b758e9515c6e6046aeb, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/4b8f385e2a6149368db5763143a57847, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/e398bf2579404175a74ffb4126afccab, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/b64702f55cd04b6c8f701e6c222cacac, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/4fed60baacfb4835bd083fcc37b44b5b, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/112f3c12b8034d378535a5c37dce88eb, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/fb47550262304a40b79ff78279ab0881, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/9aebd59358764fd7bdf53eb86d1cecc1, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/3e24b13c1d604f40897f246db374ad70, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/9ef407c96a774fce8ee62a170525a518, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/f2b51a4cc9954cffa3886e7f68f75f22, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/c9ee4ca2c333450e91e8eb32ea874f14, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/38e26fd01c104d2385109622f0f8f168, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/b819590d7295498d877248ef8197f4e6, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/b56797ec72c74028a61394f819b95cf1, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/c3714a86a64d42119e91af9d6e4f3b2f, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/cb145f814b6d437f89c05557b13e1a35, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/8f32d7a81df94d20a13bd09a516d4252, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/6416d4282af34200b62c0024e979d104, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/d5b6a76b7bd34cda81cf17d92455e828, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/5a0ef51a678545b8820c8776911d4214, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/71132b2ddcd44ab5ba2c95459eda94bd, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/011902cf4b8147d0b32e01882f87b0a2, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/8fbb834a22da43d488c1bbb40dfe3de5, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/794fa9e4c6464353b4cd53159eada40e, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/c1c83f5787744c7b80892f09ad968ea6, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/2dd95528c3be4d77a540fa906f9de38f, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/c3af42da5f9d4bc0b7b89233114673fc, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/1770377b91f242378c42b329399e0e91] to archive 2024-11-21T11:14:13,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T11:14:13,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/8b6f85bef0074899bcb1d517efe1e818 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/8b6f85bef0074899bcb1d517efe1e818 2024-11-21T11:14:13,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/fad3b6a8d0b74ab99c4cdf90aa668f83 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/fad3b6a8d0b74ab99c4cdf90aa668f83 2024-11-21T11:14:13,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/bdf34b6149e84c2299fbb4e2d3873276 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/bdf34b6149e84c2299fbb4e2d3873276 2024-11-21T11:14:13,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/31d051962e5045d280c2d236a0d65c44 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/31d051962e5045d280c2d236a0d65c44 2024-11-21T11:14:13,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/7ebaa14e3bdd4276b94cb9bd6f2f68ef to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/7ebaa14e3bdd4276b94cb9bd6f2f68ef 2024-11-21T11:14:13,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/7eaec5076bf84b758e9515c6e6046aeb to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/7eaec5076bf84b758e9515c6e6046aeb 2024-11-21T11:14:13,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/4b8f385e2a6149368db5763143a57847 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/4b8f385e2a6149368db5763143a57847 2024-11-21T11:14:13,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/e398bf2579404175a74ffb4126afccab to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/e398bf2579404175a74ffb4126afccab 2024-11-21T11:14:13,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/b64702f55cd04b6c8f701e6c222cacac to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/b64702f55cd04b6c8f701e6c222cacac 2024-11-21T11:14:13,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/4fed60baacfb4835bd083fcc37b44b5b to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/4fed60baacfb4835bd083fcc37b44b5b 2024-11-21T11:14:13,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/112f3c12b8034d378535a5c37dce88eb to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/112f3c12b8034d378535a5c37dce88eb 2024-11-21T11:14:13,894 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/fb47550262304a40b79ff78279ab0881 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/fb47550262304a40b79ff78279ab0881 2024-11-21T11:14:13,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/9aebd59358764fd7bdf53eb86d1cecc1 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/9aebd59358764fd7bdf53eb86d1cecc1 2024-11-21T11:14:13,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/3e24b13c1d604f40897f246db374ad70 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/3e24b13c1d604f40897f246db374ad70 2024-11-21T11:14:13,897 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/9ef407c96a774fce8ee62a170525a518 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/9ef407c96a774fce8ee62a170525a518 2024-11-21T11:14:13,898 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/f2b51a4cc9954cffa3886e7f68f75f22 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/f2b51a4cc9954cffa3886e7f68f75f22 2024-11-21T11:14:13,899 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/c9ee4ca2c333450e91e8eb32ea874f14 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/c9ee4ca2c333450e91e8eb32ea874f14 2024-11-21T11:14:13,900 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/38e26fd01c104d2385109622f0f8f168 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/38e26fd01c104d2385109622f0f8f168 2024-11-21T11:14:13,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/b819590d7295498d877248ef8197f4e6 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/b819590d7295498d877248ef8197f4e6 2024-11-21T11:14:13,903 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/b56797ec72c74028a61394f819b95cf1 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/b56797ec72c74028a61394f819b95cf1 2024-11-21T11:14:13,904 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/c3714a86a64d42119e91af9d6e4f3b2f to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/c3714a86a64d42119e91af9d6e4f3b2f 2024-11-21T11:14:13,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/cb145f814b6d437f89c05557b13e1a35 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/cb145f814b6d437f89c05557b13e1a35 2024-11-21T11:14:13,906 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/8f32d7a81df94d20a13bd09a516d4252 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/8f32d7a81df94d20a13bd09a516d4252 2024-11-21T11:14:13,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/6416d4282af34200b62c0024e979d104 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/6416d4282af34200b62c0024e979d104 2024-11-21T11:14:13,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/d5b6a76b7bd34cda81cf17d92455e828 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/d5b6a76b7bd34cda81cf17d92455e828 2024-11-21T11:14:13,909 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/5a0ef51a678545b8820c8776911d4214 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/5a0ef51a678545b8820c8776911d4214 2024-11-21T11:14:13,910 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/71132b2ddcd44ab5ba2c95459eda94bd to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/71132b2ddcd44ab5ba2c95459eda94bd 2024-11-21T11:14:13,911 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/011902cf4b8147d0b32e01882f87b0a2 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/011902cf4b8147d0b32e01882f87b0a2 2024-11-21T11:14:13,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/8fbb834a22da43d488c1bbb40dfe3de5 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/8fbb834a22da43d488c1bbb40dfe3de5 2024-11-21T11:14:13,913 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/794fa9e4c6464353b4cd53159eada40e to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/794fa9e4c6464353b4cd53159eada40e 2024-11-21T11:14:13,914 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/c1c83f5787744c7b80892f09ad968ea6 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/c1c83f5787744c7b80892f09ad968ea6 2024-11-21T11:14:13,915 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/2dd95528c3be4d77a540fa906f9de38f to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/2dd95528c3be4d77a540fa906f9de38f 2024-11-21T11:14:13,916 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/c3af42da5f9d4bc0b7b89233114673fc to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/c3af42da5f9d4bc0b7b89233114673fc 2024-11-21T11:14:13,917 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/1770377b91f242378c42b329399e0e91 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/1770377b91f242378c42b329399e0e91 2024-11-21T11:14:13,919 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/4f294d546e134601a7125063f955e677, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/812a707b47304e0eb57ef128c28ea027, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/9c275bef44604457b639bcc80ab3de98, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/e60244ba77f847aea0349ca4aea057b0, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/6c89c2e2b4774f9291341ed4f74b509d, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/8e7887363e914ac2b0b6902d58c1762c, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/9b6eb1806add4ddcb82d4781e433ced7, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/12231413f31f41148d654cef33302d23, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/6e232429fd6947049f9c2392061baf04, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/780be62f012d44d8965c679586cc56b7, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/6010738385404ec6a905a0d2222e4a9d, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/57e82db449fa456b9aa50abe9aae3942, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/5fff6e08bb644e7296422260088b9c7e, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/2741522c508b4efe9613e52aebe5cba2, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/2b3ef9e752ca42f5b4acd393cabd67e6, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/9c509d32e97e48b6ba5ba3d8f0b3d77f, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/b22e6aeccdfb4362b8ac7e7608c435d7, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/6d691ab7e5434ccaaa4dc353c3e8f454, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/1739e4fceb28464bb2d2b935bf13a339, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/6603fbd31f7941c4b693301542ecde2d, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/d268d8687c4c42bc9e239f81e6113895, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/cdfa1315312144ffa3f8978109fcd3e0, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/5363c2a43534485aab187292ae44bf2e, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/ce6076e5486248f5ace8469cf262075f, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/3ec8fa6ed83e417f9185f9824bf31904, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/f259879c61d24d47a386d7d25602f397, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/7745c46c9c6148c9964f8dc107c2f43b, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/d25f4caa26ec4b0ebf05c0c933e8f909, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/ca6ceacd8825467eb6ff229b019b83fc, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/e8b41f87614248c38399347f10876639, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/a441eba26d824b39a0cedd669290d0c2, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/b0e4992f32614a9784554b1fd62a68a9, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/d80d93c68e634a30bffb5c774d85fbca, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/98ea7612dd464a5787d046f806d7a720] to archive 2024-11-21T11:14:13,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T11:14:13,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/4f294d546e134601a7125063f955e677 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/4f294d546e134601a7125063f955e677 2024-11-21T11:14:13,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/812a707b47304e0eb57ef128c28ea027 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/812a707b47304e0eb57ef128c28ea027 2024-11-21T11:14:13,924 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/9c275bef44604457b639bcc80ab3de98 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/9c275bef44604457b639bcc80ab3de98 2024-11-21T11:14:13,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/e60244ba77f847aea0349ca4aea057b0 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/e60244ba77f847aea0349ca4aea057b0 2024-11-21T11:14:13,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/6c89c2e2b4774f9291341ed4f74b509d to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/6c89c2e2b4774f9291341ed4f74b509d 2024-11-21T11:14:13,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/8e7887363e914ac2b0b6902d58c1762c to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/8e7887363e914ac2b0b6902d58c1762c 2024-11-21T11:14:13,928 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/9b6eb1806add4ddcb82d4781e433ced7 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/9b6eb1806add4ddcb82d4781e433ced7 2024-11-21T11:14:13,929 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/12231413f31f41148d654cef33302d23 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/12231413f31f41148d654cef33302d23 2024-11-21T11:14:13,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/6e232429fd6947049f9c2392061baf04 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/6e232429fd6947049f9c2392061baf04 2024-11-21T11:14:13,931 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/780be62f012d44d8965c679586cc56b7 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/780be62f012d44d8965c679586cc56b7 2024-11-21T11:14:13,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/6010738385404ec6a905a0d2222e4a9d to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/6010738385404ec6a905a0d2222e4a9d 2024-11-21T11:14:13,933 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/57e82db449fa456b9aa50abe9aae3942 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/57e82db449fa456b9aa50abe9aae3942 2024-11-21T11:14:13,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/5fff6e08bb644e7296422260088b9c7e to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/5fff6e08bb644e7296422260088b9c7e 2024-11-21T11:14:13,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/2741522c508b4efe9613e52aebe5cba2 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/2741522c508b4efe9613e52aebe5cba2 2024-11-21T11:14:13,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/2b3ef9e752ca42f5b4acd393cabd67e6 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/2b3ef9e752ca42f5b4acd393cabd67e6 2024-11-21T11:14:13,938 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/9c509d32e97e48b6ba5ba3d8f0b3d77f to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/9c509d32e97e48b6ba5ba3d8f0b3d77f 2024-11-21T11:14:13,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/b22e6aeccdfb4362b8ac7e7608c435d7 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/b22e6aeccdfb4362b8ac7e7608c435d7 2024-11-21T11:14:13,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/6d691ab7e5434ccaaa4dc353c3e8f454 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/6d691ab7e5434ccaaa4dc353c3e8f454 2024-11-21T11:14:13,941 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/1739e4fceb28464bb2d2b935bf13a339 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/1739e4fceb28464bb2d2b935bf13a339 2024-11-21T11:14:13,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/6603fbd31f7941c4b693301542ecde2d to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/6603fbd31f7941c4b693301542ecde2d 2024-11-21T11:14:13,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/d268d8687c4c42bc9e239f81e6113895 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/d268d8687c4c42bc9e239f81e6113895 2024-11-21T11:14:13,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/cdfa1315312144ffa3f8978109fcd3e0 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/cdfa1315312144ffa3f8978109fcd3e0 2024-11-21T11:14:13,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/5363c2a43534485aab187292ae44bf2e to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/5363c2a43534485aab187292ae44bf2e 2024-11-21T11:14:13,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/ce6076e5486248f5ace8469cf262075f to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/ce6076e5486248f5ace8469cf262075f 2024-11-21T11:14:13,948 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/3ec8fa6ed83e417f9185f9824bf31904 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/3ec8fa6ed83e417f9185f9824bf31904 2024-11-21T11:14:13,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/f259879c61d24d47a386d7d25602f397 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/f259879c61d24d47a386d7d25602f397 2024-11-21T11:14:13,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/7745c46c9c6148c9964f8dc107c2f43b to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/7745c46c9c6148c9964f8dc107c2f43b 2024-11-21T11:14:13,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/d25f4caa26ec4b0ebf05c0c933e8f909 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/d25f4caa26ec4b0ebf05c0c933e8f909 2024-11-21T11:14:13,954 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/ca6ceacd8825467eb6ff229b019b83fc to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/ca6ceacd8825467eb6ff229b019b83fc 2024-11-21T11:14:13,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/e8b41f87614248c38399347f10876639 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/e8b41f87614248c38399347f10876639 2024-11-21T11:14:13,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/a441eba26d824b39a0cedd669290d0c2 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/a441eba26d824b39a0cedd669290d0c2 2024-11-21T11:14:13,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/b0e4992f32614a9784554b1fd62a68a9 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/b0e4992f32614a9784554b1fd62a68a9 2024-11-21T11:14:13,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/d80d93c68e634a30bffb5c774d85fbca to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/d80d93c68e634a30bffb5c774d85fbca 2024-11-21T11:14:13,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/98ea7612dd464a5787d046f806d7a720 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/98ea7612dd464a5787d046f806d7a720 2024-11-21T11:14:13,964 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/recovered.edits/544.seqid, newMaxSeqId=544, maxSeqId=1 2024-11-21T11:14:13,965 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. 2024-11-21T11:14:13,965 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for cb944dfe54fbd2ebb234cb2046a89860: Waiting for close lock at 1732187652559Running coprocessor pre-close hooks at 1732187652559Disabling compacts and flushes for region at 1732187652559Disabling writes for close at 1732187652560 (+1 ms)Obtaining lock to block concurrent updates at 1732187652560Preparing flush snapshotting stores in cb944dfe54fbd2ebb234cb2046a89860 at 1732187652560Finished memstore snapshotting TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732187652560Flushing stores of TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860. at 1732187652561 (+1 ms)Flushing cb944dfe54fbd2ebb234cb2046a89860/A: creating writer at 1732187652561Flushing cb944dfe54fbd2ebb234cb2046a89860/A: appending metadata at 1732187652565 (+4 ms)Flushing cb944dfe54fbd2ebb234cb2046a89860/A: closing flushed file at 1732187652565Flushing cb944dfe54fbd2ebb234cb2046a89860/B: creating writer at 1732187652975 (+410 ms)Flushing cb944dfe54fbd2ebb234cb2046a89860/B: appending metadata at 1732187652978 (+3 ms)Flushing cb944dfe54fbd2ebb234cb2046a89860/B: closing flushed file at 1732187652978Flushing cb944dfe54fbd2ebb234cb2046a89860/C: creating writer at 1732187653388 (+410 ms)Flushing cb944dfe54fbd2ebb234cb2046a89860/C: appending metadata at 1732187653391 (+3 ms)Flushing cb944dfe54fbd2ebb234cb2046a89860/C: closing flushed file at 1732187653391Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ed173ea: reopening flushed file at 1732187653800 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66598226: reopening flushed file at 1732187653806 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@635668dc: reopening flushed file at 1732187653811 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for cb944dfe54fbd2ebb234cb2046a89860 in 1256ms, sequenceid=541, compaction requested=true at 1732187653817 (+6 ms)Writing region close event to WAL at 1732187653961 (+144 ms)Running coprocessor post-close hooks at 1732187653965 (+4 ms)Closed at 1732187653965 2024-11-21T11:14:13,968 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:14:13,969 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=cb944dfe54fbd2ebb234cb2046a89860, regionState=CLOSED 2024-11-21T11:14:13,970 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 because future has completed 2024-11-21T11:14:13,973 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-21T11:14:13,973 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure cb944dfe54fbd2ebb234cb2046a89860, server=7b462513bfc2,33417,1732187618276 in 1.5690 sec 2024-11-21T11:14:13,975 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-21T11:14:13,975 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb944dfe54fbd2ebb234cb2046a89860, UNASSIGN in 1.5760 sec 2024-11-21T11:14:13,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-21T11:14:13,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5830 sec 2024-11-21T11:14:13,978 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732187653978"}]},"ts":"1732187653978"} 2024-11-21T11:14:13,980 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T11:14:13,980 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T11:14:13,982 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5970 sec 2024-11-21T11:14:14,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T11:14:14,536 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:14:14,539 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T11:14:14,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T11:14:14,544 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T11:14:14,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-21T11:14:14,546 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T11:14:14,550 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:14:14,554 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A, FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B, FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C, FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/recovered.edits] 2024-11-21T11:14:14,558 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/49833122549944c397854cd5825c91ba to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/49833122549944c397854cd5825c91ba 2024-11-21T11:14:14,559 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/8f168225310b46618a40cbdf5d263e8c to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/8f168225310b46618a40cbdf5d263e8c 2024-11-21T11:14:14,561 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/f8215202c81e405d965c147b59cca651 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/A/f8215202c81e405d965c147b59cca651 2024-11-21T11:14:14,563 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/370f53aaa0d94eb889d3beb80e60afe4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/370f53aaa0d94eb889d3beb80e60afe4 2024-11-21T11:14:14,564 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/ce00f5b7fdc445adafd121aef3c8f016 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/ce00f5b7fdc445adafd121aef3c8f016 2024-11-21T11:14:14,566 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/d63cf3a8ba9d42f89cc3e20f87926044 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/B/d63cf3a8ba9d42f89cc3e20f87926044 2024-11-21T11:14:14,568 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/3b5efa73d6114ee496eab4b26f2ccc2d to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/3b5efa73d6114ee496eab4b26f2ccc2d 2024-11-21T11:14:14,569 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/7fbaff4757294479ad61f4aeffee1828 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/7fbaff4757294479ad61f4aeffee1828 2024-11-21T11:14:14,571 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/996eecb2e6c7448792917a48b66c4d65 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/C/996eecb2e6c7448792917a48b66c4d65 2024-11-21T11:14:14,574 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/recovered.edits/544.seqid to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860/recovered.edits/544.seqid 2024-11-21T11:14:14,574 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/cb944dfe54fbd2ebb234cb2046a89860 2024-11-21T11:14:14,574 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T11:14:14,579 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T11:14:14,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-21T11:14:14,588 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T11:14:14,628 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T11:14:14,630 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T11:14:14,630 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T11:14:14,631 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732187654630"}]},"ts":"9223372036854775807"} 2024-11-21T11:14:14,636 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T11:14:14,636 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => cb944dfe54fbd2ebb234cb2046a89860, NAME => 'TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T11:14:14,637 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T11:14:14,637 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732187654637"}]},"ts":"9223372036854775807"} 2024-11-21T11:14:14,642 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T11:14:14,644 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T11:14:14,646 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 104 msec 2024-11-21T11:14:14,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-21T11:14:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T11:14:14,657 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:14:14,669 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 211) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;7b462513bfc2:33417-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-672728481_22 at /127.0.0.1:59582 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=395 (was 318) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5836 (was 5877) 2024-11-21T11:14:14,680 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=395, ProcessCount=11, AvailableMemoryMB=5834 2024-11-21T11:14:14,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T11:14:14,682 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T11:14:14,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T11:14:14,684 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T11:14:14,685 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:14,685 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-21T11:14:14,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-21T11:14:14,685 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T11:14:14,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741948_1124 (size=1061) 2024-11-21T11:14:14,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-21T11:14:15,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-21T11:14:15,094 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322 2024-11-21T11:14:15,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741949_1125 (size=53) 2024-11-21T11:14:15,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-21T11:14:15,501 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:15,501 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 960784f3d73443a7fc815210ff1818b4, disabling compactions & flushes 2024-11-21T11:14:15,501 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:15,501 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:15,501 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. after waiting 0 ms 2024-11-21T11:14:15,501 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:15,501 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:15,501 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 960784f3d73443a7fc815210ff1818b4: Waiting for close lock at 1732187655501Disabling compacts and flushes for region at 1732187655501Disabling writes for close at 1732187655501Writing region close event to WAL at 1732187655501Closed at 1732187655501 2024-11-21T11:14:15,502 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T11:14:15,503 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732187655502"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732187655502"}]},"ts":"1732187655502"} 2024-11-21T11:14:15,504 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T11:14:15,506 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T11:14:15,506 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732187655506"}]},"ts":"1732187655506"} 2024-11-21T11:14:15,507 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T11:14:15,508 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=960784f3d73443a7fc815210ff1818b4, ASSIGN}] 2024-11-21T11:14:15,509 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=960784f3d73443a7fc815210ff1818b4, ASSIGN 2024-11-21T11:14:15,509 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=960784f3d73443a7fc815210ff1818b4, ASSIGN; state=OFFLINE, location=7b462513bfc2,33417,1732187618276; forceNewPlan=false, retain=false 2024-11-21T11:14:15,660 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=960784f3d73443a7fc815210ff1818b4, regionState=OPENING, regionLocation=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:15,662 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=960784f3d73443a7fc815210ff1818b4, ASSIGN because future has completed 2024-11-21T11:14:15,662 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276}] 2024-11-21T11:14:15,817 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:15,818 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} 2024-11-21T11:14:15,818 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:15,818 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:15,818 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:15,818 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:15,819 INFO [StoreOpener-960784f3d73443a7fc815210ff1818b4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:15,821 INFO [StoreOpener-960784f3d73443a7fc815210ff1818b4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T11:14:15,821 INFO [StoreOpener-960784f3d73443a7fc815210ff1818b4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 960784f3d73443a7fc815210ff1818b4 columnFamilyName A 2024-11-21T11:14:15,822 DEBUG [StoreOpener-960784f3d73443a7fc815210ff1818b4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:15,822 INFO [StoreOpener-960784f3d73443a7fc815210ff1818b4-1 {}] regionserver.HStore(327): Store=960784f3d73443a7fc815210ff1818b4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:15,822 INFO [StoreOpener-960784f3d73443a7fc815210ff1818b4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:15,823 INFO [StoreOpener-960784f3d73443a7fc815210ff1818b4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T11:14:15,823 INFO [StoreOpener-960784f3d73443a7fc815210ff1818b4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 960784f3d73443a7fc815210ff1818b4 columnFamilyName B 2024-11-21T11:14:15,824 DEBUG [StoreOpener-960784f3d73443a7fc815210ff1818b4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:15,824 INFO [StoreOpener-960784f3d73443a7fc815210ff1818b4-1 {}] regionserver.HStore(327): Store=960784f3d73443a7fc815210ff1818b4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:15,824 INFO [StoreOpener-960784f3d73443a7fc815210ff1818b4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:15,825 INFO [StoreOpener-960784f3d73443a7fc815210ff1818b4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T11:14:15,825 INFO [StoreOpener-960784f3d73443a7fc815210ff1818b4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 960784f3d73443a7fc815210ff1818b4 columnFamilyName C 2024-11-21T11:14:15,825 DEBUG [StoreOpener-960784f3d73443a7fc815210ff1818b4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:15,826 INFO [StoreOpener-960784f3d73443a7fc815210ff1818b4-1 {}] regionserver.HStore(327): Store=960784f3d73443a7fc815210ff1818b4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:15,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-21T11:14:15,826 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:15,826 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:15,826 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:15,827 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:15,828 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:15,828 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:15,828 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T11:14:15,829 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:15,831 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T11:14:15,831 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 960784f3d73443a7fc815210ff1818b4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74354416, jitterRate=0.10796713829040527}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T11:14:15,831 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:15,832 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 960784f3d73443a7fc815210ff1818b4: Running coprocessor pre-open hook at 1732187655818Writing region info on filesystem at 1732187655818Initializing all the Stores at 1732187655819 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187655819Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187655819Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187655819Cleaning up temporary data from old regions at 1732187655828 (+9 ms)Running coprocessor post-open hooks at 1732187655831 (+3 ms)Region opened successfully at 1732187655832 (+1 ms) 2024-11-21T11:14:15,833 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., pid=32, masterSystemTime=1732187655814 2024-11-21T11:14:15,835 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:15,835 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:15,835 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=960784f3d73443a7fc815210ff1818b4, regionState=OPEN, openSeqNum=2, regionLocation=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:15,837 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 because future has completed 2024-11-21T11:14:15,840 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-21T11:14:15,840 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 in 176 msec 2024-11-21T11:14:15,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-21T11:14:15,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=960784f3d73443a7fc815210ff1818b4, ASSIGN in 332 msec 2024-11-21T11:14:15,842 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T11:14:15,843 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732187655842"}]},"ts":"1732187655842"} 2024-11-21T11:14:15,844 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T11:14:15,845 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T11:14:15,847 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1630 sec 2024-11-21T11:14:16,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-21T11:14:16,835 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:14:16,836 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79527845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:16,837 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:14:16,837 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:14:16,838 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:14:16,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:14:16,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:14:16,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25ac5775, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:16,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:14:16,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:14:16,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:16,841 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:14:16,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4474113, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:16,842 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:14:16,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:14:16,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:14:16,844 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:14:16,846 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T11:14:16,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 7b462513bfc2,36543,1732187617563 2024-11-21T11:14:16,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@22d64e14 2024-11-21T11:14:16,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T11:14:16,847 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T11:14:16,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T11:14:16,853 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T11:14:16,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-21T11:14:16,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741950_1126 (size=1097) 2024-11-21T11:14:17,271 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-21T11:14:17,271 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-21T11:14:17,276 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T11:14:17,282 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=960784f3d73443a7fc815210ff1818b4, REOPEN/MOVE}] 2024-11-21T11:14:17,283 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=960784f3d73443a7fc815210ff1818b4, REOPEN/MOVE 2024-11-21T11:14:17,284 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=960784f3d73443a7fc815210ff1818b4, regionState=CLOSING, regionLocation=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:17,286 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=960784f3d73443a7fc815210ff1818b4, REOPEN/MOVE because future has completed 2024-11-21T11:14:17,286 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T11:14:17,286 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276}] 2024-11-21T11:14:17,439 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:17,439 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T11:14:17,439 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing 960784f3d73443a7fc815210ff1818b4, disabling compactions & flushes 2024-11-21T11:14:17,439 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:17,439 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:17,439 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. after waiting 0 ms 2024-11-21T11:14:17,439 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:17,443 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T11:14:17,443 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:17,443 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for 960784f3d73443a7fc815210ff1818b4: Waiting for close lock at 1732187657439Running coprocessor pre-close hooks at 1732187657439Disabling compacts and flushes for region at 1732187657439Disabling writes for close at 1732187657439Writing region close event to WAL at 1732187657440 (+1 ms)Running coprocessor post-close hooks at 1732187657443 (+3 ms)Closed at 1732187657443 2024-11-21T11:14:17,444 WARN [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: 960784f3d73443a7fc815210ff1818b4 to self. 2024-11-21T11:14:17,446 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:17,446 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=960784f3d73443a7fc815210ff1818b4, regionState=CLOSED 2024-11-21T11:14:17,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 because future has completed 2024-11-21T11:14:17,450 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-21T11:14:17,450 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 in 163 msec 2024-11-21T11:14:17,451 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=960784f3d73443a7fc815210ff1818b4, REOPEN/MOVE; state=CLOSED, location=7b462513bfc2,33417,1732187618276; forceNewPlan=false, retain=true 2024-11-21T11:14:17,601 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=960784f3d73443a7fc815210ff1818b4, regionState=OPENING, regionLocation=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:17,603 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=960784f3d73443a7fc815210ff1818b4, REOPEN/MOVE because future has completed 2024-11-21T11:14:17,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276}] 2024-11-21T11:14:17,759 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:17,759 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} 2024-11-21T11:14:17,760 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:17,760 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:17,760 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:17,760 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:17,764 INFO [StoreOpener-960784f3d73443a7fc815210ff1818b4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:17,765 INFO [StoreOpener-960784f3d73443a7fc815210ff1818b4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T11:14:17,770 INFO [StoreOpener-960784f3d73443a7fc815210ff1818b4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 960784f3d73443a7fc815210ff1818b4 columnFamilyName A 2024-11-21T11:14:17,772 DEBUG [StoreOpener-960784f3d73443a7fc815210ff1818b4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:17,772 INFO [StoreOpener-960784f3d73443a7fc815210ff1818b4-1 {}] regionserver.HStore(327): Store=960784f3d73443a7fc815210ff1818b4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:17,773 INFO [StoreOpener-960784f3d73443a7fc815210ff1818b4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:17,773 INFO [StoreOpener-960784f3d73443a7fc815210ff1818b4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T11:14:17,773 INFO [StoreOpener-960784f3d73443a7fc815210ff1818b4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 960784f3d73443a7fc815210ff1818b4 columnFamilyName B 2024-11-21T11:14:17,773 DEBUG [StoreOpener-960784f3d73443a7fc815210ff1818b4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:17,774 INFO [StoreOpener-960784f3d73443a7fc815210ff1818b4-1 {}] regionserver.HStore(327): Store=960784f3d73443a7fc815210ff1818b4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:17,774 INFO [StoreOpener-960784f3d73443a7fc815210ff1818b4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:17,774 INFO [StoreOpener-960784f3d73443a7fc815210ff1818b4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T11:14:17,775 INFO [StoreOpener-960784f3d73443a7fc815210ff1818b4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 960784f3d73443a7fc815210ff1818b4 columnFamilyName C 2024-11-21T11:14:17,775 DEBUG [StoreOpener-960784f3d73443a7fc815210ff1818b4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:17,775 INFO [StoreOpener-960784f3d73443a7fc815210ff1818b4-1 {}] regionserver.HStore(327): Store=960784f3d73443a7fc815210ff1818b4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:17,775 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:17,775 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:17,776 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:17,777 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:17,777 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:17,777 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:17,778 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T11:14:17,779 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:17,780 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 960784f3d73443a7fc815210ff1818b4; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74131095, jitterRate=0.10463939607143402}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T11:14:17,780 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:17,781 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 960784f3d73443a7fc815210ff1818b4: Running coprocessor pre-open hook at 1732187657760Writing region info on filesystem at 1732187657760Initializing all the Stores at 1732187657761 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187657761Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187657761Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187657761Cleaning up temporary data from old regions at 1732187657777 (+16 ms)Running coprocessor post-open hooks at 1732187657780 (+3 ms)Region opened successfully at 1732187657781 (+1 ms) 2024-11-21T11:14:17,782 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., pid=37, masterSystemTime=1732187657756 2024-11-21T11:14:17,784 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:17,784 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:17,784 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=960784f3d73443a7fc815210ff1818b4, regionState=OPEN, openSeqNum=5, regionLocation=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:17,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 because future has completed 2024-11-21T11:14:17,788 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-21T11:14:17,788 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 in 182 msec 2024-11-21T11:14:17,790 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-21T11:14:17,790 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=960784f3d73443a7fc815210ff1818b4, REOPEN/MOVE in 506 msec 2024-11-21T11:14:17,792 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-21T11:14:17,792 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 516 msec 2024-11-21T11:14:17,795 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 938 msec 2024-11-21T11:14:17,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-21T11:14:17,797 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:14:17,797 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5adc4e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:17,797 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:14:17,797 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:14:17,799 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:14:17,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:14:17,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:14:17,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f7f88a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:17,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:14:17,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:14:17,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:17,800 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:14:17,801 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78276902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:17,801 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681c9b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:17,801 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:14:17,801 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:14:17,802 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:14:17,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:14:17,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:14:17,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@579718f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:17,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:14:17,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:14:17,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:17,803 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:14:17,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50cb75b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:17,805 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51d91e8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:17,805 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:14:17,805 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:14:17,806 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:14:17,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:14:17,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:14:17,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@112d4998, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:17,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:14:17,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:14:17,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:17,807 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:14:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51a9b02b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:17,808 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771f5e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:17,808 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:14:17,808 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:14:17,809 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:14:17,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:14:17,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:14:17,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b5e5b9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:17,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:14:17,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:14:17,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:17,810 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:14:17,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f36d1a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:17,811 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b75e2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:17,811 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:14:17,811 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:14:17,812 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:14:17,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:14:17,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:14:17,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24cefbe8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:17,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:14:17,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:14:17,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:17,813 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:14:17,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36a97364, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:17,814 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5453a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:17,815 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:14:17,815 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:14:17,815 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:14:17,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:14:17,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:14:17,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d861a86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:17,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:14:17,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:14:17,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:17,816 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:14:17,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63a1f7f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:17,817 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b6e690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:17,817 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:14:17,817 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:14:17,818 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:14:17,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:14:17,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:14:17,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@320638f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:17,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:14:17,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:14:17,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:17,819 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:14:17,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34c0b19c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:17,820 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd05ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:17,820 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:14:17,820 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:14:17,821 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:14:17,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:14:17,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:14:17,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2baaac90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:17,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:14:17,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:14:17,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:17,822 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:14:17,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d9c4167, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:17,823 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dff183a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:17,823 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:14:17,823 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:14:17,824 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:14:17,824 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:14:17,824 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:14:17,824 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4653e1f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:17,824 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:14:17,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:14:17,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:17,825 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35654, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:14:17,826 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7405a174, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:17,827 DEBUG [Thread-643 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:14:17,827 DEBUG [Thread-652 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:14:17,827 DEBUG [Thread-641 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:14:17,827 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:14:17,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:14:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:14:17,828 DEBUG [Thread-648 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:14:17,828 DEBUG [Thread-635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:14:17,828 DEBUG [Thread-646 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:14:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:14:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:14:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:14:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:14:17,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:14:17,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:14:17,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:14:17,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:14:17,829 DEBUG [Thread-637 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:14:17,829 DEBUG [Thread-650 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:14:17,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-21T11:14:17,830 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:14:17,830 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:14:17,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-21T11:14:17,830 DEBUG [Thread-639 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:14:17,831 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:14:17,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:14:17,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:14:17,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:14:17,831 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:14:17,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:14:17,831 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60504, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:14:17,832 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:14:17,832 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:14:17,832 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:14:17,832 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:14:17,832 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:14:17,832 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:14:17,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:14:17,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:14:17,833 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:14:17,833 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:14:17,834 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:14:17,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5] 2024-11-21T11:14:17,836 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5] 2024-11-21T11:14:17,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5] 2024-11-21T11:14:17,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5] 2024-11-21T11:14:17,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5] 2024-11-21T11:14:17,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5] 2024-11-21T11:14:17,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5] 2024-11-21T11:14:17,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5] 2024-11-21T11:14:17,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5] 2024-11-21T11:14:17,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:17,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 960784f3d73443a7fc815210ff1818b4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T11:14:17,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=A 2024-11-21T11:14:17,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:17,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=B 2024-11-21T11:14:17,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:17,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=C 2024-11-21T11:14:17,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:17,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732187657881, max timestamp=1732187657892Num uniques 859; 2024-11-21T11:14:17,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732187657881, max timestamp=1732187657892Num uniques 859; 2024-11-21T11:14:17,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-21T11:14:17,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732187657881, max timestamp=1732187657892Num uniques 859; 2024-11-21T11:14:17,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-21T11:14:17,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-21T11:14:17,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c23c76f6ea22409dac96cfd87c314f77_960784f3d73443a7fc815210ff1818b4 is 50, key is test_row_0/A:col10/1732187657846/Put/seqid=0 2024-11-21T11:14:17,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:17,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187717919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:17,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:17,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:17,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:17,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187717919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:17,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:17,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:17,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:17,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:17,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187717920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:17,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:17,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:17,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:17,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:17,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187717921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:17,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:17,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:17,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:17,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:17,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-21T11:14:17,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732187717929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:17,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:17,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:17,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:17,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741951_1127 (size=12154) 2024-11-21T11:14:17,944 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:17,950 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c23c76f6ea22409dac96cfd87c314f77_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c23c76f6ea22409dac96cfd87c314f77_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:17,952 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/cba0e1b407754cf79812527522409715, store: [table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:17,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/cba0e1b407754cf79812527522409715 is 175, key is test_row_0/A:col10/1732187657846/Put/seqid=0 2024-11-21T11:14:17,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741952_1128 (size=30955) 2024-11-21T11:14:17,975 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/cba0e1b407754cf79812527522409715 2024-11-21T11:14:17,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-21T11:14:17,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:17,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:17,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:17,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:17,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:18,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/094f3cb0ae3e4c5994c15e6a1dee530b is 50, key is test_row_0/B:col10/1732187657846/Put/seqid=0 2024-11-21T11:14:18,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741953_1129 (size=12001) 2024-11-21T11:14:18,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/094f3cb0ae3e4c5994c15e6a1dee530b 2024-11-21T11:14:18,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:18,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187718036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:18,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:18,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:18,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187718036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:18,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:18,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:18,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187718036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:18,041 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:18,041 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,041 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:18,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187718036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:18,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:18,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:18,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732187718046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:18,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:18,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/42102e60828f43a8935f025b596fa4e6 is 50, key is test_row_0/C:col10/1732187657846/Put/seqid=0 2024-11-21T11:14:18,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741954_1130 (size=12001) 2024-11-21T11:14:18,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/42102e60828f43a8935f025b596fa4e6 2024-11-21T11:14:18,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/cba0e1b407754cf79812527522409715 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/cba0e1b407754cf79812527522409715 2024-11-21T11:14:18,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/cba0e1b407754cf79812527522409715, entries=150, sequenceid=15, filesize=30.2 K 2024-11-21T11:14:18,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/094f3cb0ae3e4c5994c15e6a1dee530b as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/094f3cb0ae3e4c5994c15e6a1dee530b 2024-11-21T11:14:18,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/094f3cb0ae3e4c5994c15e6a1dee530b, entries=150, sequenceid=15, filesize=11.7 K 2024-11-21T11:14:18,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/42102e60828f43a8935f025b596fa4e6 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/42102e60828f43a8935f025b596fa4e6 2024-11-21T11:14:18,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/42102e60828f43a8935f025b596fa4e6, entries=150, sequenceid=15, filesize=11.7 K 2024-11-21T11:14:18,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 960784f3d73443a7fc815210ff1818b4 in 269ms, sequenceid=15, compaction requested=false 2024-11-21T11:14:18,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:18,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-21T11:14:18,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:18,142 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 960784f3d73443a7fc815210ff1818b4 3/3 column families, dataSize=134.18 KB heapSize=366.23 KB 2024-11-21T11:14:18,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=A 2024-11-21T11:14:18,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:14:18,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=B 2024-11-21T11:14:18,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:14:18,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=C 2024-11-21T11:14:18,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:14:18,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-21T11:14:18,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112169f23f98e538415cbc7f42d9d6daf483_960784f3d73443a7fc815210ff1818b4 is 50, key is test_row_0/A:col10/1732187657905/Put/seqid=0 2024-11-21T11:14:18,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741955_1131 (size=12154) 2024-11-21T11:14:18,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:18,182 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112169f23f98e538415cbc7f42d9d6daf483_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112169f23f98e538415cbc7f42d9d6daf483_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:18,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/f9e198556ddb4cc688a6ea95880bae58, store: [table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:18,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/f9e198556ddb4cc688a6ea95880bae58 is 175, key is test_row_0/A:col10/1732187657905/Put/seqid=0 2024-11-21T11:14:18,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741956_1132 (size=30955) 2024-11-21T11:14:18,199 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=48, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/f9e198556ddb4cc688a6ea95880bae58 2024-11-21T11:14:18,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/927beefde1e64b81987b3f5fbf8757ab is 50, key is test_row_0/B:col10/1732187657905/Put/seqid=0 2024-11-21T11:14:18,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741957_1133 (size=12001) 2024-11-21T11:14:18,225 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/927beefde1e64b81987b3f5fbf8757ab 2024-11-21T11:14:18,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/ff20ef03f31e4225aa231e07f1c40422 is 50, key is test_row_0/C:col10/1732187657905/Put/seqid=0 2024-11-21T11:14:18,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:18,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:18,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741958_1134 (size=12001) 2024-11-21T11:14:18,260 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/ff20ef03f31e4225aa231e07f1c40422 2024-11-21T11:14:18,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:18,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187718256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:18,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:18,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187718258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:18,262 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,263 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,263 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:18,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732187718258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:18,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:18,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187718260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:18,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:18,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/f9e198556ddb4cc688a6ea95880bae58 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/f9e198556ddb4cc688a6ea95880bae58 2024-11-21T11:14:18,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:18,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187718262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:18,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:18,279 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/f9e198556ddb4cc688a6ea95880bae58, entries=150, sequenceid=48, filesize=30.2 K 2024-11-21T11:14:18,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/927beefde1e64b81987b3f5fbf8757ab as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/927beefde1e64b81987b3f5fbf8757ab 2024-11-21T11:14:18,288 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/927beefde1e64b81987b3f5fbf8757ab, entries=150, sequenceid=48, filesize=11.7 K 2024-11-21T11:14:18,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/ff20ef03f31e4225aa231e07f1c40422 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/ff20ef03f31e4225aa231e07f1c40422 2024-11-21T11:14:18,296 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/ff20ef03f31e4225aa231e07f1c40422, entries=150, sequenceid=48, filesize=11.7 K 2024-11-21T11:14:18,297 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~366.19 KB/374976, currentSize=53.67 KB/54960 for 960784f3d73443a7fc815210ff1818b4 in 155ms, sequenceid=48, compaction requested=false 2024-11-21T11:14:18,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:18,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:18,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-21T11:14:18,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-21T11:14:18,302 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-21T11:14:18,302 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 467 msec 2024-11-21T11:14:18,304 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 475 msec 2024-11-21T11:14:18,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:18,370 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 960784f3d73443a7fc815210ff1818b4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-21T11:14:18,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=A 2024-11-21T11:14:18,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:18,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=B 2024-11-21T11:14:18,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:18,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=C 2024-11-21T11:14:18,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:18,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121654de44d62104e1082a69700f67e1ee2_960784f3d73443a7fc815210ff1818b4 is 50, key is test_row_0/A:col10/1732187658370/Put/seqid=0 2024-11-21T11:14:18,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741959_1135 (size=14594) 2024-11-21T11:14:18,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-21T11:14:18,455 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:14:18,456 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:14:18,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-21T11:14:18,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T11:14:18,459 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:14:18,461 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:14:18,461 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:14:18,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:18,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187718453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:18,466 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,466 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,466 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:18,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:18,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187718464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:18,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:18,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:18,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187718466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:18,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:18,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187718466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:18,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:18,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:18,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T11:14:18,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187718576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:18,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:18,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732187718576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:18,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:18,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:18,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187718576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:18,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:18,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:18,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:18,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187718576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:18,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:18,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187718577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:18,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:18,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:18,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T11:14:18,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:18,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:18,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:18,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:18,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:18,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:18,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T11:14:18,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:18,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:18,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:18,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:18,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T11:14:18,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:18,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187718785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:18,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:18,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187718785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:18,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:18,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:18,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187718786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:18,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:18,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:18,796 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:18,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:18,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187718796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:18,801 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121654de44d62104e1082a69700f67e1ee2_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121654de44d62104e1082a69700f67e1ee2_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:18,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:18,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:18,804 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/4cfd88eb5ba849b684c9c5eb3b027f6b, store: [table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:18,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/4cfd88eb5ba849b684c9c5eb3b027f6b is 175, key is test_row_0/A:col10/1732187658370/Put/seqid=0 2024-11-21T11:14:18,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741960_1136 (size=39549) 2024-11-21T11:14:18,814 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/4cfd88eb5ba849b684c9c5eb3b027f6b 2024-11-21T11:14:18,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/499658ddf97143f7bfce661b2cf81e3e is 50, key is test_row_0/B:col10/1732187658370/Put/seqid=0 2024-11-21T11:14:18,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741961_1137 (size=12001) 2024-11-21T11:14:18,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/499658ddf97143f7bfce661b2cf81e3e 2024-11-21T11:14:18,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/aa8ccd886e674201ba62b442c463f505 is 50, key is test_row_0/C:col10/1732187658370/Put/seqid=0 2024-11-21T11:14:18,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741962_1138 (size=12001) 2024-11-21T11:14:18,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/aa8ccd886e674201ba62b442c463f505 2024-11-21T11:14:18,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/4cfd88eb5ba849b684c9c5eb3b027f6b as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/4cfd88eb5ba849b684c9c5eb3b027f6b 2024-11-21T11:14:18,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/4cfd88eb5ba849b684c9c5eb3b027f6b, entries=200, sequenceid=61, filesize=38.6 K 2024-11-21T11:14:18,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/499658ddf97143f7bfce661b2cf81e3e as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/499658ddf97143f7bfce661b2cf81e3e 2024-11-21T11:14:18,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/499658ddf97143f7bfce661b2cf81e3e, entries=150, sequenceid=61, filesize=11.7 K 2024-11-21T11:14:18,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/aa8ccd886e674201ba62b442c463f505 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/aa8ccd886e674201ba62b442c463f505 2024-11-21T11:14:18,901 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/aa8ccd886e674201ba62b442c463f505, entries=150, sequenceid=61, filesize=11.7 K 2024-11-21T11:14:18,902 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 960784f3d73443a7fc815210ff1818b4 in 532ms, sequenceid=61, compaction requested=true 2024-11-21T11:14:18,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:18,902 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:14:18,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 960784f3d73443a7fc815210ff1818b4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:14:18,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:18,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 960784f3d73443a7fc815210ff1818b4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:14:18,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:14:18,903 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:14:18,903 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 960784f3d73443a7fc815210ff1818b4/A is initiating minor compaction (all files) 2024-11-21T11:14:18,903 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 960784f3d73443a7fc815210ff1818b4/A in TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:18,904 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/cba0e1b407754cf79812527522409715, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/f9e198556ddb4cc688a6ea95880bae58, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/4cfd88eb5ba849b684c9c5eb3b027f6b] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp, totalSize=99.1 K 2024-11-21T11:14:18,904 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:18,904 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. files: [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/cba0e1b407754cf79812527522409715, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/f9e198556ddb4cc688a6ea95880bae58, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/4cfd88eb5ba849b684c9c5eb3b027f6b] 2024-11-21T11:14:18,905 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting cba0e1b407754cf79812527522409715, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732187657845 2024-11-21T11:14:18,905 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9e198556ddb4cc688a6ea95880bae58, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732187657883 2024-11-21T11:14:18,906 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:14:18,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 960784f3d73443a7fc815210ff1818b4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:14:18,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:14:18,907 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cfd88eb5ba849b684c9c5eb3b027f6b, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732187658253 2024-11-21T11:14:18,908 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:14:18,908 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 960784f3d73443a7fc815210ff1818b4/B is initiating minor compaction (all files) 2024-11-21T11:14:18,908 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 960784f3d73443a7fc815210ff1818b4/B in TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:18,909 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/094f3cb0ae3e4c5994c15e6a1dee530b, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/927beefde1e64b81987b3f5fbf8757ab, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/499658ddf97143f7bfce661b2cf81e3e] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp, totalSize=35.2 K 2024-11-21T11:14:18,909 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 094f3cb0ae3e4c5994c15e6a1dee530b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732187657845 2024-11-21T11:14:18,910 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 927beefde1e64b81987b3f5fbf8757ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732187657883 2024-11-21T11:14:18,911 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 499658ddf97143f7bfce661b2cf81e3e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732187658253 2024-11-21T11:14:18,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T11:14:18,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:18,923 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 960784f3d73443a7fc815210ff1818b4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-21T11:14:18,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=A 2024-11-21T11:14:18,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:18,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=B 2024-11-21T11:14:18,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:18,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=C 2024-11-21T11:14:18,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:18,929 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:18,938 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121185b322641a740d7afd3af12339a0505_960784f3d73443a7fc815210ff1818b4 store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:18,955 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 960784f3d73443a7fc815210ff1818b4#B#compaction#121 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:18,956 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/0dbac1aba0f94ca087ae636116231d9a is 50, key is test_row_0/B:col10/1732187658370/Put/seqid=0 2024-11-21T11:14:18,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b96a20c796d041ad8f8f463ae76dcd7f_960784f3d73443a7fc815210ff1818b4 is 50, key is test_row_0/A:col10/1732187658458/Put/seqid=0 2024-11-21T11:14:18,961 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121185b322641a740d7afd3af12339a0505_960784f3d73443a7fc815210ff1818b4, store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:18,961 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121185b322641a740d7afd3af12339a0505_960784f3d73443a7fc815210ff1818b4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:19,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741964_1140 (size=12154) 2024-11-21T11:14:19,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:19,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741963_1139 (size=12104) 2024-11-21T11:14:19,015 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b96a20c796d041ad8f8f463ae76dcd7f_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b96a20c796d041ad8f8f463ae76dcd7f_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:19,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741965_1141 (size=4469) 2024-11-21T11:14:19,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/7f81958bf0d84d178cd4dbc9831b9b15, store: [table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:19,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/7f81958bf0d84d178cd4dbc9831b9b15 is 175, key is test_row_0/A:col10/1732187658458/Put/seqid=0 2024-11-21T11:14:19,018 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 960784f3d73443a7fc815210ff1818b4#A#compaction#120 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:19,019 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/6f5a18fcfb6a4174bf8d53ee665b309a is 175, key is test_row_0/A:col10/1732187658370/Put/seqid=0 2024-11-21T11:14:19,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741966_1142 (size=30955) 2024-11-21T11:14:19,041 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/7f81958bf0d84d178cd4dbc9831b9b15 2024-11-21T11:14:19,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741967_1143 (size=31058) 2024-11-21T11:14:19,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/5fdf229b277c461fa30e78e48ee61220 is 50, key is test_row_0/B:col10/1732187658458/Put/seqid=0 2024-11-21T11:14:19,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741968_1144 (size=12001) 2024-11-21T11:14:19,084 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/5fdf229b277c461fa30e78e48ee61220 2024-11-21T11:14:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T11:14:19,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/2c15022308e3474baa8038f9a3efd5a6 is 50, key is test_row_0/C:col10/1732187658458/Put/seqid=0 2024-11-21T11:14:19,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:19,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:19,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:19,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732187719111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:19,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:19,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:19,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187719114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:19,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:19,125 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,125 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187719114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:19,125 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:19,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:19,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:19,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187719116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:19,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:19,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:19,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187719117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:19,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:19,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741969_1145 (size=12001) 2024-11-21T11:14:19,132 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/2c15022308e3474baa8038f9a3efd5a6 2024-11-21T11:14:19,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/7f81958bf0d84d178cd4dbc9831b9b15 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/7f81958bf0d84d178cd4dbc9831b9b15 2024-11-21T11:14:19,148 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/7f81958bf0d84d178cd4dbc9831b9b15, entries=150, sequenceid=81, filesize=30.2 K 2024-11-21T11:14:19,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/5fdf229b277c461fa30e78e48ee61220 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/5fdf229b277c461fa30e78e48ee61220 2024-11-21T11:14:19,156 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/5fdf229b277c461fa30e78e48ee61220, entries=150, sequenceid=81, filesize=11.7 K 2024-11-21T11:14:19,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/2c15022308e3474baa8038f9a3efd5a6 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/2c15022308e3474baa8038f9a3efd5a6 2024-11-21T11:14:19,162 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/2c15022308e3474baa8038f9a3efd5a6, entries=150, sequenceid=81, filesize=11.7 K 2024-11-21T11:14:19,163 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 960784f3d73443a7fc815210ff1818b4 in 240ms, sequenceid=81, compaction requested=true 2024-11-21T11:14:19,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:19,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:19,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-21T11:14:19,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-21T11:14:19,172 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-21T11:14:19,173 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 705 msec 2024-11-21T11:14:19,175 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 717 msec 2024-11-21T11:14:19,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:19,228 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 960784f3d73443a7fc815210ff1818b4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-21T11:14:19,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=A 2024-11-21T11:14:19,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:19,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=B 2024-11-21T11:14:19,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:19,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=C 2024-11-21T11:14:19,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:19,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121cb7084c83b404433ba811b52e087bc84_960784f3d73443a7fc815210ff1818b4 is 50, key is test_row_0/A:col10/1732187659112/Put/seqid=0 2024-11-21T11:14:19,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741970_1146 (size=14594) 2024-11-21T11:14:19,271 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:19,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:19,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187719268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:19,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:19,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187719270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:19,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:19,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:19,279 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121cb7084c83b404433ba811b52e087bc84_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121cb7084c83b404433ba811b52e087bc84_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:19,280 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/17e9f486798a411e875de69412baadf3, store: [table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:19,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/17e9f486798a411e875de69412baadf3 is 175, key is test_row_0/A:col10/1732187659112/Put/seqid=0 2024-11-21T11:14:19,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:19,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187719273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:19,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:19,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187719276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:19,287 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,287 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:19,287 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:19,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732187719276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:19,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:19,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:19,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741971_1147 (size=39549) 2024-11-21T11:14:19,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:19,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187719385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:19,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:19,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187719385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:19,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:19,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:19,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:19,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732187719397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:19,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:19,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:19,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187719397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:19,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:19,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187719397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:19,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:19,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:19,420 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/0dbac1aba0f94ca087ae636116231d9a as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/0dbac1aba0f94ca087ae636116231d9a 2024-11-21T11:14:19,434 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 960784f3d73443a7fc815210ff1818b4/B of 960784f3d73443a7fc815210ff1818b4 into 0dbac1aba0f94ca087ae636116231d9a(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:19,434 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:19,434 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., storeName=960784f3d73443a7fc815210ff1818b4/B, priority=13, startTime=1732187658903; duration=0sec 2024-11-21T11:14:19,434 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:14:19,434 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 960784f3d73443a7fc815210ff1818b4:B 2024-11-21T11:14:19,434 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T11:14:19,436 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T11:14:19,436 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 960784f3d73443a7fc815210ff1818b4/C is initiating minor compaction (all files) 2024-11-21T11:14:19,436 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 960784f3d73443a7fc815210ff1818b4/C in TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:19,436 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/42102e60828f43a8935f025b596fa4e6, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/ff20ef03f31e4225aa231e07f1c40422, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/aa8ccd886e674201ba62b442c463f505, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/2c15022308e3474baa8038f9a3efd5a6] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp, totalSize=46.9 K 2024-11-21T11:14:19,437 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 42102e60828f43a8935f025b596fa4e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732187657845 2024-11-21T11:14:19,437 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting ff20ef03f31e4225aa231e07f1c40422, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732187657883 2024-11-21T11:14:19,438 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting aa8ccd886e674201ba62b442c463f505, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732187658253 2024-11-21T11:14:19,438 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c15022308e3474baa8038f9a3efd5a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732187658458 2024-11-21T11:14:19,454 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 960784f3d73443a7fc815210ff1818b4#C#compaction#126 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:19,455 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/84a1a0c23f9147b287a7801daf152fc4 is 50, key is test_row_0/C:col10/1732187658458/Put/seqid=0 2024-11-21T11:14:19,469 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/6f5a18fcfb6a4174bf8d53ee665b309a as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/6f5a18fcfb6a4174bf8d53ee665b309a 2024-11-21T11:14:19,478 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 960784f3d73443a7fc815210ff1818b4/A of 960784f3d73443a7fc815210ff1818b4 into 6f5a18fcfb6a4174bf8d53ee665b309a(size=30.3 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:19,478 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:19,478 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., storeName=960784f3d73443a7fc815210ff1818b4/A, priority=13, startTime=1732187658902; duration=0sec 2024-11-21T11:14:19,478 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:19,478 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 960784f3d73443a7fc815210ff1818b4:A 2024-11-21T11:14:19,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741972_1148 (size=12139) 2024-11-21T11:14:19,495 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/84a1a0c23f9147b287a7801daf152fc4 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/84a1a0c23f9147b287a7801daf152fc4 2024-11-21T11:14:19,506 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 960784f3d73443a7fc815210ff1818b4/C of 960784f3d73443a7fc815210ff1818b4 into 84a1a0c23f9147b287a7801daf152fc4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:19,506 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:19,506 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., storeName=960784f3d73443a7fc815210ff1818b4/C, priority=12, startTime=1732187658903; duration=0sec 2024-11-21T11:14:19,506 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:19,506 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 960784f3d73443a7fc815210ff1818b4:C 2024-11-21T11:14:19,511 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T11:14:19,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T11:14:19,596 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:14:19,597 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:14:19,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-21T11:14:19,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T11:14:19,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:19,600 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:14:19,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187719596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:19,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:19,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187719596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:19,600 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:14:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,601 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:14:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:19,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:19,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732187719605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:19,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:19,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:19,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187719605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:19,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:19,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187719606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:19,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:19,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:19,698 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/17e9f486798a411e875de69412baadf3 2024-11-21T11:14:19,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T11:14:19,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/174b528c8da044a09a92de9b56dd3b00 is 50, key is test_row_0/B:col10/1732187659112/Put/seqid=0 2024-11-21T11:14:19,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741973_1149 (size=12001) 2024-11-21T11:14:19,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T11:14:19,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:19,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:19,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:19,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:19,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:19,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:19,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T11:14:19,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:19,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:19,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:19,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:19,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:19,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:19,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:19,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187719905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:19,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:19,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187719905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:19,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:19,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:19,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T11:14:19,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:19,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732187719915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:19,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:19,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187719916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:19,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:19,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:19,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187719916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:19,925 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,925 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,925 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:19,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:19,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:20,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T11:14:20,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:20,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:20,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:20,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:20,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:20,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:20,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/174b528c8da044a09a92de9b56dd3b00 2024-11-21T11:14:20,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/aed09e60274243b2a257f6d112a6534d is 50, key is test_row_0/C:col10/1732187659112/Put/seqid=0 2024-11-21T11:14:20,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741974_1150 (size=12001) 2024-11-21T11:14:20,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T11:14:20,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:20,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:20,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:20,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:20,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:20,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:20,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T11:14:20,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T11:14:20,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:20,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:20,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:20,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:20,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:20,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:20,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:20,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187720415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:20,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:20,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:20,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187720415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:20,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:20,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:20,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:20,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:20,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:20,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732187720436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:20,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:20,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:20,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:20,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:20,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187720436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:20,439 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:20,439 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:20,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:20,439 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:20,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187720436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:20,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T11:14:20,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:20,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:20,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:20,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:20,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:20,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:20,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/aed09e60274243b2a257f6d112a6534d 2024-11-21T11:14:20,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/17e9f486798a411e875de69412baadf3 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/17e9f486798a411e875de69412baadf3 2024-11-21T11:14:20,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/17e9f486798a411e875de69412baadf3, entries=200, sequenceid=94, filesize=38.6 K 2024-11-21T11:14:20,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/174b528c8da044a09a92de9b56dd3b00 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/174b528c8da044a09a92de9b56dd3b00 2024-11-21T11:14:20,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/174b528c8da044a09a92de9b56dd3b00, entries=150, sequenceid=94, filesize=11.7 K 2024-11-21T11:14:20,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/aed09e60274243b2a257f6d112a6534d as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/aed09e60274243b2a257f6d112a6534d 2024-11-21T11:14:20,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/aed09e60274243b2a257f6d112a6534d, entries=150, sequenceid=94, filesize=11.7 K 2024-11-21T11:14:20,563 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 960784f3d73443a7fc815210ff1818b4 in 1335ms, sequenceid=94, compaction requested=true 2024-11-21T11:14:20,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:20,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 960784f3d73443a7fc815210ff1818b4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:14:20,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:20,563 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:14:20,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 960784f3d73443a7fc815210ff1818b4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:14:20,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:20,563 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:14:20,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 960784f3d73443a7fc815210ff1818b4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:14:20,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:14:20,564 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:14:20,564 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 960784f3d73443a7fc815210ff1818b4/A is initiating minor compaction (all files) 2024-11-21T11:14:20,565 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 960784f3d73443a7fc815210ff1818b4/A in TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:20,565 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:14:20,565 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/6f5a18fcfb6a4174bf8d53ee665b309a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/7f81958bf0d84d178cd4dbc9831b9b15, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/17e9f486798a411e875de69412baadf3] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp, totalSize=99.2 K 2024-11-21T11:14:20,565 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 960784f3d73443a7fc815210ff1818b4/B is initiating minor compaction (all files) 2024-11-21T11:14:20,565 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:20,565 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 960784f3d73443a7fc815210ff1818b4/B in TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:20,565 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. files: [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/6f5a18fcfb6a4174bf8d53ee665b309a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/7f81958bf0d84d178cd4dbc9831b9b15, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/17e9f486798a411e875de69412baadf3] 2024-11-21T11:14:20,565 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/0dbac1aba0f94ca087ae636116231d9a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/5fdf229b277c461fa30e78e48ee61220, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/174b528c8da044a09a92de9b56dd3b00] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp, totalSize=35.3 K 2024-11-21T11:14:20,565 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f5a18fcfb6a4174bf8d53ee665b309a, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732187658253 2024-11-21T11:14:20,565 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dbac1aba0f94ca087ae636116231d9a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732187658253 2024-11-21T11:14:20,566 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fdf229b277c461fa30e78e48ee61220, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732187658458 2024-11-21T11:14:20,566 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f81958bf0d84d178cd4dbc9831b9b15, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732187658458 2024-11-21T11:14:20,567 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 174b528c8da044a09a92de9b56dd3b00, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732187659112 2024-11-21T11:14:20,567 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17e9f486798a411e875de69412baadf3, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732187659112 2024-11-21T11:14:20,574 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:20,576 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 960784f3d73443a7fc815210ff1818b4#B#compaction#130 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:20,576 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/bdb56f0412eb4952863af668a88fd324 is 50, key is test_row_0/B:col10/1732187659112/Put/seqid=0 2024-11-21T11:14:20,576 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121ca1066104c7b40dfbfccc58192ad93eb_960784f3d73443a7fc815210ff1818b4 store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:20,580 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121ca1066104c7b40dfbfccc58192ad93eb_960784f3d73443a7fc815210ff1818b4, store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:20,580 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ca1066104c7b40dfbfccc58192ad93eb_960784f3d73443a7fc815210ff1818b4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:20,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741975_1151 (size=12207) 2024-11-21T11:14:20,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741976_1152 (size=4469) 2024-11-21T11:14:20,594 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/bdb56f0412eb4952863af668a88fd324 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/bdb56f0412eb4952863af668a88fd324 2024-11-21T11:14:20,596 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 960784f3d73443a7fc815210ff1818b4#A#compaction#129 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:20,597 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/26aa176b776b4f739f05a40e8d2dd61a is 175, key is test_row_0/A:col10/1732187659112/Put/seqid=0 2024-11-21T11:14:20,602 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 960784f3d73443a7fc815210ff1818b4/B of 960784f3d73443a7fc815210ff1818b4 into bdb56f0412eb4952863af668a88fd324(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:20,602 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:20,603 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., storeName=960784f3d73443a7fc815210ff1818b4/B, priority=13, startTime=1732187660563; duration=0sec 2024-11-21T11:14:20,603 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:14:20,603 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 960784f3d73443a7fc815210ff1818b4:B 2024-11-21T11:14:20,603 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-21T11:14:20,604 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-21T11:14:20,604 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-21T11:14:20,604 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. because compaction request was cancelled 2024-11-21T11:14:20,604 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 960784f3d73443a7fc815210ff1818b4:C 2024-11-21T11:14:20,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741977_1153 (size=31161) 2024-11-21T11:14:20,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T11:14:20,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:20,676 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 960784f3d73443a7fc815210ff1818b4 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T11:14:20,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=A 2024-11-21T11:14:20,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:20,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=B 2024-11-21T11:14:20,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:20,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=C 2024-11-21T11:14:20,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:20,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112107c86f81f69b46508680bbab79737037_960784f3d73443a7fc815210ff1818b4 is 50, key is test_row_0/A:col10/1732187659272/Put/seqid=0 2024-11-21T11:14:20,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741978_1154 (size=12154) 2024-11-21T11:14:20,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T11:14:21,015 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/26aa176b776b4f739f05a40e8d2dd61a as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/26aa176b776b4f739f05a40e8d2dd61a 2024-11-21T11:14:21,023 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 960784f3d73443a7fc815210ff1818b4/A of 960784f3d73443a7fc815210ff1818b4 into 26aa176b776b4f739f05a40e8d2dd61a(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:21,023 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:21,023 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., storeName=960784f3d73443a7fc815210ff1818b4/A, priority=13, startTime=1732187660563; duration=0sec 2024-11-21T11:14:21,023 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:21,023 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 960784f3d73443a7fc815210ff1818b4:A 2024-11-21T11:14:21,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:21,106 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112107c86f81f69b46508680bbab79737037_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112107c86f81f69b46508680bbab79737037_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:21,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/07ce0aa3fd6e401fa9184db6ec8f34ff, store: [table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:21,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/07ce0aa3fd6e401fa9184db6ec8f34ff is 175, key is test_row_0/A:col10/1732187659272/Put/seqid=0 2024-11-21T11:14:21,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741979_1155 (size=30955) 2024-11-21T11:14:21,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:21,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:21,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:21,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187721448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:21,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:21,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:21,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187721449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:21,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:21,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:21,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:21,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:21,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:21,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187721451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:21,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:21,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:21,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:21,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:21,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732187721455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:21,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:21,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187721455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:21,460 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_0' in 960784f3d73443a7fc815210ff1818b4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:14:21,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:21,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:21,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:21,460 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_2' in 960784f3d73443a7fc815210ff1818b4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:14:21,461 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:21,461 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:21,461 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:21,520 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/07ce0aa3fd6e401fa9184db6ec8f34ff 2024-11-21T11:14:21,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/a7419d2819ef462db99eeb9194d75b13 is 50, key is test_row_0/B:col10/1732187659272/Put/seqid=0 2024-11-21T11:14:21,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741980_1156 (size=12001) 2024-11-21T11:14:21,541 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/a7419d2819ef462db99eeb9194d75b13 2024-11-21T11:14:21,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/d73949bcd52a4500ba8ce8f0f0f41217 is 50, key is test_row_0/C:col10/1732187659272/Put/seqid=0 2024-11-21T11:14:21,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:21,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187721555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:21,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:21,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:21,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:21,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741981_1157 (size=12001) 2024-11-21T11:14:21,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:21,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187721566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:21,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:21,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:21,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:21,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187721566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:21,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:21,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:21,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:21,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:21,697 INFO [master/7b462513bfc2:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-21T11:14:21,697 INFO [master/7b462513bfc2:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-21T11:14:21,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T11:14:21,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:21,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187721765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:21,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:21,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:21,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:21,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:21,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187721776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:21,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:21,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187721776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:21,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:21,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:21,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:21,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:21,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:21,959 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/d73949bcd52a4500ba8ce8f0f0f41217 2024-11-21T11:14:21,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/07ce0aa3fd6e401fa9184db6ec8f34ff as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/07ce0aa3fd6e401fa9184db6ec8f34ff 2024-11-21T11:14:21,970 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/07ce0aa3fd6e401fa9184db6ec8f34ff, entries=150, sequenceid=117, filesize=30.2 K 2024-11-21T11:14:21,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/a7419d2819ef462db99eeb9194d75b13 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/a7419d2819ef462db99eeb9194d75b13 2024-11-21T11:14:21,978 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/a7419d2819ef462db99eeb9194d75b13, entries=150, sequenceid=117, filesize=11.7 K 2024-11-21T11:14:21,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/d73949bcd52a4500ba8ce8f0f0f41217 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/d73949bcd52a4500ba8ce8f0f0f41217 2024-11-21T11:14:21,984 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/d73949bcd52a4500ba8ce8f0f0f41217, entries=150, sequenceid=117, filesize=11.7 K 2024-11-21T11:14:21,986 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 960784f3d73443a7fc815210ff1818b4 in 1311ms, sequenceid=117, compaction requested=true 2024-11-21T11:14:21,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:21,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:21,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-21T11:14:21,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-21T11:14:21,991 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-21T11:14:21,991 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3870 sec 2024-11-21T11:14:21,994 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.3950 sec 2024-11-21T11:14:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:22,090 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 960784f3d73443a7fc815210ff1818b4 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T11:14:22,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=A 2024-11-21T11:14:22,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:22,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=B 2024-11-21T11:14:22,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:22,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=C 2024-11-21T11:14:22,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:22,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411213b210e9151094a088422d5dc1cc28b3d_960784f3d73443a7fc815210ff1818b4 is 50, key is test_row_0/A:col10/1732187661447/Put/seqid=0 2024-11-21T11:14:22,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:22,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187722114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:22,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:22,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:22,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:22,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:22,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187722116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:22,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:22,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:22,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:22,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:22,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187722126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:22,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:22,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:22,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:22,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741982_1158 (size=12254) 2024-11-21T11:14:22,130 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:22,136 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411213b210e9151094a088422d5dc1cc28b3d_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213b210e9151094a088422d5dc1cc28b3d_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:22,137 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/94c591f009de46d09b2083006ecc478d, store: [table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:22,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/94c591f009de46d09b2083006ecc478d is 175, key is test_row_0/A:col10/1732187661447/Put/seqid=0 2024-11-21T11:14:22,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741983_1159 (size=31055) 2024-11-21T11:14:22,152 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/94c591f009de46d09b2083006ecc478d 2024-11-21T11:14:22,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/e103786a84c543e3912ba1a9093a7a1d is 50, key is test_row_0/B:col10/1732187661447/Put/seqid=0 2024-11-21T11:14:22,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741984_1160 (size=12101) 2024-11-21T11:14:22,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187722236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:22,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:22,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187722236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:22,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:22,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:22,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187722236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:22,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:22,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:22,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:22,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:22,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:22,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:22,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187722446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:22,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:22,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187722446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:22,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:22,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:22,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187722446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:22,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:22,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:22,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:22,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:22,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:22,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:22,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/e103786a84c543e3912ba1a9093a7a1d 2024-11-21T11:14:22,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/b735e166509d4d2891b2ddc1a6aa3baf is 50, key is test_row_0/C:col10/1732187661447/Put/seqid=0 2024-11-21T11:14:22,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741985_1161 (size=12101) 2024-11-21T11:14:22,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/b735e166509d4d2891b2ddc1a6aa3baf 2024-11-21T11:14:22,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/94c591f009de46d09b2083006ecc478d as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/94c591f009de46d09b2083006ecc478d 2024-11-21T11:14:22,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/94c591f009de46d09b2083006ecc478d, entries=150, sequenceid=134, filesize=30.3 K 2024-11-21T11:14:22,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/e103786a84c543e3912ba1a9093a7a1d as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/e103786a84c543e3912ba1a9093a7a1d 2024-11-21T11:14:22,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/e103786a84c543e3912ba1a9093a7a1d, entries=150, sequenceid=134, filesize=11.8 K 2024-11-21T11:14:22,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/b735e166509d4d2891b2ddc1a6aa3baf as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/b735e166509d4d2891b2ddc1a6aa3baf 2024-11-21T11:14:22,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/b735e166509d4d2891b2ddc1a6aa3baf, entries=150, sequenceid=134, filesize=11.8 K 2024-11-21T11:14:22,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 960784f3d73443a7fc815210ff1818b4 in 547ms, sequenceid=134, compaction requested=true 2024-11-21T11:14:22,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:22,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 960784f3d73443a7fc815210ff1818b4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:14:22,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:22,638 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:14:22,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 960784f3d73443a7fc815210ff1818b4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:14:22,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:14:22,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 960784f3d73443a7fc815210ff1818b4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:14:22,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T11:14:22,638 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T11:14:22,639 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93171 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:14:22,639 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 960784f3d73443a7fc815210ff1818b4/A is initiating minor compaction (all files) 2024-11-21T11:14:22,639 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 960784f3d73443a7fc815210ff1818b4/A in TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:22,639 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/26aa176b776b4f739f05a40e8d2dd61a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/07ce0aa3fd6e401fa9184db6ec8f34ff, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/94c591f009de46d09b2083006ecc478d] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp, totalSize=91.0 K 2024-11-21T11:14:22,639 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:22,639 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T11:14:22,639 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. files: [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/26aa176b776b4f739f05a40e8d2dd61a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/07ce0aa3fd6e401fa9184db6ec8f34ff, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/94c591f009de46d09b2083006ecc478d] 2024-11-21T11:14:22,639 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 960784f3d73443a7fc815210ff1818b4/C is initiating minor compaction (all files) 2024-11-21T11:14:22,639 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 960784f3d73443a7fc815210ff1818b4/C in TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:22,639 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/84a1a0c23f9147b287a7801daf152fc4, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/aed09e60274243b2a257f6d112a6534d, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/d73949bcd52a4500ba8ce8f0f0f41217, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/b735e166509d4d2891b2ddc1a6aa3baf] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp, totalSize=47.1 K 2024-11-21T11:14:22,640 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26aa176b776b4f739f05a40e8d2dd61a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732187659112 2024-11-21T11:14:22,640 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 84a1a0c23f9147b287a7801daf152fc4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732187658458 2024-11-21T11:14:22,640 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07ce0aa3fd6e401fa9184db6ec8f34ff, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732187659263 2024-11-21T11:14:22,641 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94c591f009de46d09b2083006ecc478d, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732187661447 2024-11-21T11:14:22,641 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting aed09e60274243b2a257f6d112a6534d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732187659112 2024-11-21T11:14:22,642 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting d73949bcd52a4500ba8ce8f0f0f41217, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732187659263 2024-11-21T11:14:22,643 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting b735e166509d4d2891b2ddc1a6aa3baf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732187661447 2024-11-21T11:14:22,653 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:22,661 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121b2e8ec2d7da44d3b944c08d1f0f35566_960784f3d73443a7fc815210ff1818b4 store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:22,665 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121b2e8ec2d7da44d3b944c08d1f0f35566_960784f3d73443a7fc815210ff1818b4, store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:22,665 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b2e8ec2d7da44d3b944c08d1f0f35566_960784f3d73443a7fc815210ff1818b4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:22,675 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 960784f3d73443a7fc815210ff1818b4#C#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:22,676 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/cf3e8570320747bab12b770692e4e1c4 is 50, key is test_row_0/C:col10/1732187661447/Put/seqid=0 2024-11-21T11:14:22,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741987_1163 (size=12375) 2024-11-21T11:14:22,694 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/cf3e8570320747bab12b770692e4e1c4 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/cf3e8570320747bab12b770692e4e1c4 2024-11-21T11:14:22,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741986_1162 (size=4469) 2024-11-21T11:14:22,707 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 960784f3d73443a7fc815210ff1818b4/C of 960784f3d73443a7fc815210ff1818b4 into cf3e8570320747bab12b770692e4e1c4(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:22,707 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:22,707 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., storeName=960784f3d73443a7fc815210ff1818b4/C, priority=12, startTime=1732187662638; duration=0sec 2024-11-21T11:14:22,708 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:14:22,708 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 960784f3d73443a7fc815210ff1818b4:C 2024-11-21T11:14:22,708 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:14:22,710 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:14:22,710 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 960784f3d73443a7fc815210ff1818b4/B is initiating minor compaction (all files) 2024-11-21T11:14:22,710 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 960784f3d73443a7fc815210ff1818b4/B in TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:22,710 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/bdb56f0412eb4952863af668a88fd324, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/a7419d2819ef462db99eeb9194d75b13, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/e103786a84c543e3912ba1a9093a7a1d] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp, totalSize=35.5 K 2024-11-21T11:14:22,711 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting bdb56f0412eb4952863af668a88fd324, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732187659112 2024-11-21T11:14:22,711 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting a7419d2819ef462db99eeb9194d75b13, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732187659263 2024-11-21T11:14:22,715 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting e103786a84c543e3912ba1a9093a7a1d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732187661447 2024-11-21T11:14:22,726 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 960784f3d73443a7fc815210ff1818b4#B#compaction#139 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:22,727 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/759319bb5f7b4ce7a4957850b31798b3 is 50, key is test_row_0/B:col10/1732187661447/Put/seqid=0 2024-11-21T11:14:22,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741988_1164 (size=12409) 2024-11-21T11:14:22,742 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/759319bb5f7b4ce7a4957850b31798b3 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/759319bb5f7b4ce7a4957850b31798b3 2024-11-21T11:14:22,747 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 960784f3d73443a7fc815210ff1818b4/B of 960784f3d73443a7fc815210ff1818b4 into 759319bb5f7b4ce7a4957850b31798b3(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:22,747 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:22,747 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., storeName=960784f3d73443a7fc815210ff1818b4/B, priority=13, startTime=1732187662638; duration=0sec 2024-11-21T11:14:22,748 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:22,748 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 960784f3d73443a7fc815210ff1818b4:B 2024-11-21T11:14:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:22,758 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 960784f3d73443a7fc815210ff1818b4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-21T11:14:22,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=A 2024-11-21T11:14:22,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:22,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=B 2024-11-21T11:14:22,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:22,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=C 2024-11-21T11:14:22,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:22,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121846e8aceff0f4c9b8adb756de188fe4c_960784f3d73443a7fc815210ff1818b4 is 50, key is test_row_0/A:col10/1732187662108/Put/seqid=0 2024-11-21T11:14:22,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741989_1165 (size=12304) 2024-11-21T11:14:22,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187722807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:22,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187722814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:22,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:22,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:22,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187722815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:22,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:22,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:22,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:22,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:22,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187722925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:22,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:22,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187722926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187722926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:22,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:22,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:22,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:23,101 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 960784f3d73443a7fc815210ff1818b4#A#compaction#137 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:23,102 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/fed607b2986245c79ec041225ae14e5a is 175, key is test_row_0/A:col10/1732187661447/Put/seqid=0 2024-11-21T11:14:23,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741990_1166 (size=31363) 2024-11-21T11:14:23,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187723136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:23,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:23,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:23,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:23,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187723145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:23,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187723145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:23,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:23,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:23,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:23,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:23,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:23,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:23,179 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:23,189 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121846e8aceff0f4c9b8adb756de188fe4c_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121846e8aceff0f4c9b8adb756de188fe4c_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:23,191 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/91c0c5e9a44a4f4eb1bc67b470185999, store: [table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:23,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/91c0c5e9a44a4f4eb1bc67b470185999 is 175, key is test_row_0/A:col10/1732187662108/Put/seqid=0 2024-11-21T11:14:23,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741991_1167 (size=31105) 2024-11-21T11:14:23,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:23,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187723446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:23,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:23,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:23,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:23,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:23,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187723465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:23,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:23,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:23,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:23,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:23,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187723465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:23,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:23,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:23,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:23,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:23,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187723476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:23,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:23,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732187723476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:23,482 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_2' in 960784f3d73443a7fc815210ff1818b4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:14:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:23,484 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_0' in 960784f3d73443a7fc815210ff1818b4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:14:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:23,540 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/fed607b2986245c79ec041225ae14e5a as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/fed607b2986245c79ec041225ae14e5a 2024-11-21T11:14:23,551 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 960784f3d73443a7fc815210ff1818b4/A of 960784f3d73443a7fc815210ff1818b4 into fed607b2986245c79ec041225ae14e5a(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:23,551 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:23,551 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., storeName=960784f3d73443a7fc815210ff1818b4/A, priority=13, startTime=1732187662637; duration=0sec 2024-11-21T11:14:23,551 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:23,552 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 960784f3d73443a7fc815210ff1818b4:A 2024-11-21T11:14:23,599 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/91c0c5e9a44a4f4eb1bc67b470185999 2024-11-21T11:14:23,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/0a7b91bf6038477cbd1c9b0502b47144 is 50, key is test_row_0/B:col10/1732187662108/Put/seqid=0 2024-11-21T11:14:23,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741992_1168 (size=12151) 2024-11-21T11:14:23,631 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/0a7b91bf6038477cbd1c9b0502b47144 2024-11-21T11:14:23,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/33a3a64823a34fef82311c3164d154b5 is 50, key is test_row_0/C:col10/1732187662108/Put/seqid=0 2024-11-21T11:14:23,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741993_1169 (size=12151) 2024-11-21T11:14:23,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T11:14:23,767 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:14:23,767 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:14:23,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-21T11:14:23,771 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:14:23,772 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:14:23,773 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:14:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T11:14:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T11:14:23,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T11:14:23,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:23,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:23,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:23,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:23,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:23,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187723956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:23,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:23,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:23,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:23,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187723975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:23,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187723975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:24,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/33a3a64823a34fef82311c3164d154b5 2024-11-21T11:14:24,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T11:14:24,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:24,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:24,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:24,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:24,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:24,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T11:14:24,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/91c0c5e9a44a4f4eb1bc67b470185999 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/91c0c5e9a44a4f4eb1bc67b470185999 2024-11-21T11:14:24,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/91c0c5e9a44a4f4eb1bc67b470185999, entries=150, sequenceid=153, filesize=30.4 K 2024-11-21T11:14:24,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/0a7b91bf6038477cbd1c9b0502b47144 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/0a7b91bf6038477cbd1c9b0502b47144 2024-11-21T11:14:24,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/0a7b91bf6038477cbd1c9b0502b47144, entries=150, sequenceid=153, filesize=11.9 K 2024-11-21T11:14:24,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/33a3a64823a34fef82311c3164d154b5 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/33a3a64823a34fef82311c3164d154b5 2024-11-21T11:14:24,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/33a3a64823a34fef82311c3164d154b5, entries=150, sequenceid=153, filesize=11.9 K 2024-11-21T11:14:24,154 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 960784f3d73443a7fc815210ff1818b4 in 1366ms, sequenceid=153, compaction requested=false 2024-11-21T11:14:24,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:24,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T11:14:24,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:24,235 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 960784f3d73443a7fc815210ff1818b4 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-21T11:14:24,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=A 2024-11-21T11:14:24,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:24,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=B 2024-11-21T11:14:24,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:24,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=C 2024-11-21T11:14:24,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:24,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f99a629553d647789d0d06956188e114_960784f3d73443a7fc815210ff1818b4 is 50, key is test_row_0/A:col10/1732187662813/Put/seqid=0 2024-11-21T11:14:24,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741994_1170 (size=12304) 2024-11-21T11:14:24,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:24,265 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f99a629553d647789d0d06956188e114_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f99a629553d647789d0d06956188e114_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:24,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/e5e0a73b00d546cbac6f165fc226f8a9, store: [table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:24,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/e5e0a73b00d546cbac6f165fc226f8a9 is 175, key is test_row_0/A:col10/1732187662813/Put/seqid=0 2024-11-21T11:14:24,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741995_1171 (size=31105) 2024-11-21T11:14:24,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T11:14:24,697 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/e5e0a73b00d546cbac6f165fc226f8a9 2024-11-21T11:14:24,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/c34858f38d554857a9eeea1b5cc22e8a is 50, key is test_row_0/B:col10/1732187662813/Put/seqid=0 2024-11-21T11:14:24,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741996_1172 (size=12151) 2024-11-21T11:14:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T11:14:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:24,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:25,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:25,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187725019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:25,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:25,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187725024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:25,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:25,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:25,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:25,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:25,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:25,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:25,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:25,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187725026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:25,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:25,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:25,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:25,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:25,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187725135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:25,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:25,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:25,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187725135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:25,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:25,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:25,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:25,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:25,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:25,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187725135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:25,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:25,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:25,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:25,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:25,189 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/c34858f38d554857a9eeea1b5cc22e8a 2024-11-21T11:14:25,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/0a2218f2f5754b8281e9a0a28b4b5622 is 50, key is test_row_0/C:col10/1732187662813/Put/seqid=0 2024-11-21T11:14:25,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741997_1173 (size=12151) 2024-11-21T11:14:25,228 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/0a2218f2f5754b8281e9a0a28b4b5622 2024-11-21T11:14:25,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/e5e0a73b00d546cbac6f165fc226f8a9 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/e5e0a73b00d546cbac6f165fc226f8a9 2024-11-21T11:14:25,242 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/e5e0a73b00d546cbac6f165fc226f8a9, entries=150, sequenceid=169, filesize=30.4 K 2024-11-21T11:14:25,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/c34858f38d554857a9eeea1b5cc22e8a as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/c34858f38d554857a9eeea1b5cc22e8a 2024-11-21T11:14:25,262 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/c34858f38d554857a9eeea1b5cc22e8a, entries=150, sequenceid=169, filesize=11.9 K 2024-11-21T11:14:25,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/0a2218f2f5754b8281e9a0a28b4b5622 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/0a2218f2f5754b8281e9a0a28b4b5622 2024-11-21T11:14:25,269 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/0a2218f2f5754b8281e9a0a28b4b5622, entries=150, sequenceid=169, filesize=11.9 K 2024-11-21T11:14:25,270 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 960784f3d73443a7fc815210ff1818b4 in 1035ms, sequenceid=169, compaction requested=true 2024-11-21T11:14:25,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:25,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:25,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-21T11:14:25,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-21T11:14:25,274 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-21T11:14:25,275 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4990 sec 2024-11-21T11:14:25,277 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.5070 sec 2024-11-21T11:14:25,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:25,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 960784f3d73443a7fc815210ff1818b4 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T11:14:25,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=A 2024-11-21T11:14:25,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:25,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=B 2024-11-21T11:14:25,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:25,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=C 2024-11-21T11:14:25,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:25,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121fda071f51c61468fbd37f8edd517a6ad_960784f3d73443a7fc815210ff1818b4 is 50, key is test_row_0/A:col10/1732187665347/Put/seqid=0 2024-11-21T11:14:25,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:25,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187725367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:25,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:25,376 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:25,376 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:25,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187725369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:25,376 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:25,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:25,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:25,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:25,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:25,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187725376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:25,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:25,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:25,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:25,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741998_1174 (size=17284) 2024-11-21T11:14:25,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:25,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187725486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:25,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:25,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:25,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187725486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:25,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:25,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:25,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:25,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187725486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:25,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:25,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:25,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:25,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:25,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:25,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:25,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:25,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187725695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187725696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:25,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:25,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:25,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187725696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:25,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:25,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:25,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:25,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:25,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:25,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:25,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:25,785 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:25,789 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121fda071f51c61468fbd37f8edd517a6ad_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121fda071f51c61468fbd37f8edd517a6ad_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:25,790 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/1d80620e283c4e0d89fe0304a03b52d4, store: [table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:25,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/1d80620e283c4e0d89fe0304a03b52d4 is 175, key is test_row_0/A:col10/1732187665347/Put/seqid=0 2024-11-21T11:14:25,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741999_1175 (size=48389) 2024-11-21T11:14:25,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T11:14:25,916 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:14:25,916 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:14:25,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-21T11:14:25,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T11:14:25,919 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:14:25,920 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:14:25,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:14:26,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:26,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187726015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:26,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:26,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187726015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:26,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:26,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187726016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:26,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T11:14:26,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T11:14:26,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:26,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:26,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:26,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:26,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:26,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:26,199 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=189, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/1d80620e283c4e0d89fe0304a03b52d4 2024-11-21T11:14:26,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/89460b4dac574c7aab9e31041e645023 is 50, key is test_row_0/B:col10/1732187665347/Put/seqid=0 2024-11-21T11:14:26,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742000_1176 (size=12151) 2024-11-21T11:14:26,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/89460b4dac574c7aab9e31041e645023 2024-11-21T11:14:26,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T11:14:26,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:26,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:26,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:26,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:26,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T11:14:26,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/a6df1aab4af84d48975f29771c7309a8 is 50, key is test_row_0/C:col10/1732187665347/Put/seqid=0 2024-11-21T11:14:26,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742001_1177 (size=12151) 2024-11-21T11:14:26,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/a6df1aab4af84d48975f29771c7309a8 2024-11-21T11:14:26,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/1d80620e283c4e0d89fe0304a03b52d4 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/1d80620e283c4e0d89fe0304a03b52d4 2024-11-21T11:14:26,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/1d80620e283c4e0d89fe0304a03b52d4, entries=250, sequenceid=189, filesize=47.3 K 2024-11-21T11:14:26,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/89460b4dac574c7aab9e31041e645023 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/89460b4dac574c7aab9e31041e645023 2024-11-21T11:14:26,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/89460b4dac574c7aab9e31041e645023, entries=150, sequenceid=189, filesize=11.9 K 2024-11-21T11:14:26,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/a6df1aab4af84d48975f29771c7309a8 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/a6df1aab4af84d48975f29771c7309a8 2024-11-21T11:14:26,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/a6df1aab4af84d48975f29771c7309a8, entries=150, sequenceid=189, filesize=11.9 K 2024-11-21T11:14:26,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 960784f3d73443a7fc815210ff1818b4 in 949ms, sequenceid=189, compaction requested=true 2024-11-21T11:14:26,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:26,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 960784f3d73443a7fc815210ff1818b4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:14:26,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:26,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 960784f3d73443a7fc815210ff1818b4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:14:26,298 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T11:14:26,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:26,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 960784f3d73443a7fc815210ff1818b4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:14:26,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:14:26,298 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T11:14:26,299 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141962 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T11:14:26,299 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 960784f3d73443a7fc815210ff1818b4/A is initiating minor compaction (all files) 2024-11-21T11:14:26,300 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 960784f3d73443a7fc815210ff1818b4/A in TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:26,300 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/fed607b2986245c79ec041225ae14e5a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/91c0c5e9a44a4f4eb1bc67b470185999, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/e5e0a73b00d546cbac6f165fc226f8a9, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/1d80620e283c4e0d89fe0304a03b52d4] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp, totalSize=138.6 K 2024-11-21T11:14:26,300 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:26,300 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. files: [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/fed607b2986245c79ec041225ae14e5a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/91c0c5e9a44a4f4eb1bc67b470185999, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/e5e0a73b00d546cbac6f165fc226f8a9, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/1d80620e283c4e0d89fe0304a03b52d4] 2024-11-21T11:14:26,300 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T11:14:26,300 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 960784f3d73443a7fc815210ff1818b4/B is initiating minor compaction (all files) 2024-11-21T11:14:26,300 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 960784f3d73443a7fc815210ff1818b4/B in TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:26,301 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/759319bb5f7b4ce7a4957850b31798b3, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/0a7b91bf6038477cbd1c9b0502b47144, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/c34858f38d554857a9eeea1b5cc22e8a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/89460b4dac574c7aab9e31041e645023] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp, totalSize=47.7 K 2024-11-21T11:14:26,304 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 759319bb5f7b4ce7a4957850b31798b3, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732187661447 2024-11-21T11:14:26,304 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting fed607b2986245c79ec041225ae14e5a, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732187661447 2024-11-21T11:14:26,304 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a7b91bf6038477cbd1c9b0502b47144, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732187662108 2024-11-21T11:14:26,305 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91c0c5e9a44a4f4eb1bc67b470185999, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732187662108 2024-11-21T11:14:26,305 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting c34858f38d554857a9eeea1b5cc22e8a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732187662801 2024-11-21T11:14:26,305 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5e0a73b00d546cbac6f165fc226f8a9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732187662801 2024-11-21T11:14:26,305 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 89460b4dac574c7aab9e31041e645023, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732187665005 2024-11-21T11:14:26,305 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d80620e283c4e0d89fe0304a03b52d4, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732187665005 2024-11-21T11:14:26,318 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:26,320 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 960784f3d73443a7fc815210ff1818b4#B#compaction#150 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:26,321 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/aa3c24978c0547a88c36825bb4691b1d is 50, key is test_row_0/B:col10/1732187665347/Put/seqid=0 2024-11-21T11:14:26,321 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121e3ddb67959ed449e8c615f2e1d20469c_960784f3d73443a7fc815210ff1818b4 store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:26,325 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121e3ddb67959ed449e8c615f2e1d20469c_960784f3d73443a7fc815210ff1818b4, store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:26,325 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e3ddb67959ed449e8c615f2e1d20469c_960784f3d73443a7fc815210ff1818b4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:26,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742002_1178 (size=12595) 2024-11-21T11:14:26,338 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/aa3c24978c0547a88c36825bb4691b1d as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/aa3c24978c0547a88c36825bb4691b1d 2024-11-21T11:14:26,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742003_1179 (size=4469) 2024-11-21T11:14:26,351 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 960784f3d73443a7fc815210ff1818b4/B of 960784f3d73443a7fc815210ff1818b4 into aa3c24978c0547a88c36825bb4691b1d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:26,351 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:26,351 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., storeName=960784f3d73443a7fc815210ff1818b4/B, priority=12, startTime=1732187666298; duration=0sec 2024-11-21T11:14:26,351 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:14:26,351 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 960784f3d73443a7fc815210ff1818b4:B 2024-11-21T11:14:26,352 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T11:14:26,352 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 960784f3d73443a7fc815210ff1818b4#A#compaction#149 average throughput is 0.72 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:26,353 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/e822c0660da149a7a69040e77aa51296 is 175, key is test_row_0/A:col10/1732187665347/Put/seqid=0 2024-11-21T11:14:26,353 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48828 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T11:14:26,353 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 960784f3d73443a7fc815210ff1818b4/C is initiating minor compaction (all files) 2024-11-21T11:14:26,353 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 960784f3d73443a7fc815210ff1818b4/C in TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:26,354 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/cf3e8570320747bab12b770692e4e1c4, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/33a3a64823a34fef82311c3164d154b5, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/0a2218f2f5754b8281e9a0a28b4b5622, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/a6df1aab4af84d48975f29771c7309a8] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp, totalSize=47.7 K 2024-11-21T11:14:26,354 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting cf3e8570320747bab12b770692e4e1c4, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732187661447 2024-11-21T11:14:26,355 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 33a3a64823a34fef82311c3164d154b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732187662108 2024-11-21T11:14:26,355 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a2218f2f5754b8281e9a0a28b4b5622, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732187662801 2024-11-21T11:14:26,355 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting a6df1aab4af84d48975f29771c7309a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732187665005 2024-11-21T11:14:26,371 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 960784f3d73443a7fc815210ff1818b4#C#compaction#151 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:26,372 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/bb262f01070c4c268b052b613972475d is 50, key is test_row_0/C:col10/1732187665347/Put/seqid=0 2024-11-21T11:14:26,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742004_1180 (size=31549) 2024-11-21T11:14:26,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T11:14:26,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:26,383 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 960784f3d73443a7fc815210ff1818b4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-21T11:14:26,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=A 2024-11-21T11:14:26,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:26,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=B 2024-11-21T11:14:26,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:26,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=C 2024-11-21T11:14:26,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:26,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121eed1378065b14199ba543bc7a7e2d35f_960784f3d73443a7fc815210ff1818b4 is 50, key is test_row_0/A:col10/1732187665365/Put/seqid=0 2024-11-21T11:14:26,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742005_1181 (size=12561) 2024-11-21T11:14:26,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742006_1182 (size=12304) 2024-11-21T11:14:26,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:26,452 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121eed1378065b14199ba543bc7a7e2d35f_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121eed1378065b14199ba543bc7a7e2d35f_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:26,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/29b8abe74cb44a61b4939dd5a63101e8, store: [table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:26,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/29b8abe74cb44a61b4939dd5a63101e8 is 175, key is test_row_0/A:col10/1732187665365/Put/seqid=0 2024-11-21T11:14:26,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742007_1183 (size=31105) 2024-11-21T11:14:26,504 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=202, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/29b8abe74cb44a61b4939dd5a63101e8 2024-11-21T11:14:26,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/94c15e07870d41d885d7e3892f1bf62c is 50, key is test_row_0/B:col10/1732187665365/Put/seqid=0 2024-11-21T11:14:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:26,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:26,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742008_1184 (size=12151) 2024-11-21T11:14:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T11:14:26,550 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/94c15e07870d41d885d7e3892f1bf62c 2024-11-21T11:14:26,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/02b3c9440844466a800778e818203568 is 50, key is test_row_0/C:col10/1732187665365/Put/seqid=0 2024-11-21T11:14:26,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742009_1185 (size=12151) 2024-11-21T11:14:26,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:26,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187726578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:26,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:26,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:26,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:26,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:26,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187726582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:26,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:26,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:26,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:26,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:26,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187726590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:26,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:26,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:26,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:26,594 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/02b3c9440844466a800778e818203568 2024-11-21T11:14:26,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/29b8abe74cb44a61b4939dd5a63101e8 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/29b8abe74cb44a61b4939dd5a63101e8 2024-11-21T11:14:26,605 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/29b8abe74cb44a61b4939dd5a63101e8, entries=150, sequenceid=202, filesize=30.4 K 2024-11-21T11:14:26,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/94c15e07870d41d885d7e3892f1bf62c as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/94c15e07870d41d885d7e3892f1bf62c 2024-11-21T11:14:26,613 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/94c15e07870d41d885d7e3892f1bf62c, entries=150, sequenceid=202, filesize=11.9 K 2024-11-21T11:14:26,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/02b3c9440844466a800778e818203568 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/02b3c9440844466a800778e818203568 2024-11-21T11:14:26,625 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/02b3c9440844466a800778e818203568, entries=150, sequenceid=202, filesize=11.9 K 2024-11-21T11:14:26,626 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 960784f3d73443a7fc815210ff1818b4 in 243ms, sequenceid=202, compaction requested=false 2024-11-21T11:14:26,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:26,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:26,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-21T11:14:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-21T11:14:26,630 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-21T11:14:26,630 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 708 msec 2024-11-21T11:14:26,633 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 715 msec 2024-11-21T11:14:26,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732187666569, max timestamp=1732187666698Num uniques 859; 2024-11-21T11:14:26,703 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732187666569, max timestamp=1732187666698Num uniques 859; 2024-11-21T11:14:26,703 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T11:14:26,703 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T11:14:26,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732187666569, max timestamp=1732187666698Num uniques 859; 2024-11-21T11:14:26,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T11:14:26,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 960784f3d73443a7fc815210ff1818b4 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-11-21T11:14:26,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=A 2024-11-21T11:14:26,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:14:26,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=B 2024-11-21T11:14:26,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:14:26,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=C 2024-11-21T11:14:26,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:14:26,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:26,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112171e7eb530ed04ad28df7fe2040c7ed92_960784f3d73443a7fc815210ff1818b4 is 50, key is test_row_0/A:col10/1732187666723/Put/seqid=0 2024-11-21T11:14:26,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742010_1186 (size=14794) 2024-11-21T11:14:26,772 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:26,787 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112171e7eb530ed04ad28df7fe2040c7ed92_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112171e7eb530ed04ad28df7fe2040c7ed92_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:26,788 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/e8d36b01d135480897fc379a13e8b6c3, store: [table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:26,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/e8d36b01d135480897fc379a13e8b6c3 is 175, key is test_row_0/A:col10/1732187666723/Put/seqid=0 2024-11-21T11:14:26,791 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/e822c0660da149a7a69040e77aa51296 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/e822c0660da149a7a69040e77aa51296 2024-11-21T11:14:26,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732187666760, max timestamp=1732187666789Num uniques 859; 2024-11-21T11:14:26,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732187666760, max timestamp=1732187666789Num uniques 859; 2024-11-21T11:14:26,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T11:14:26,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T11:14:26,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732187666760, max timestamp=1732187666789Num uniques 859; 2024-11-21T11:14:26,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T11:14:26,800 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 960784f3d73443a7fc815210ff1818b4/A of 960784f3d73443a7fc815210ff1818b4 into e822c0660da149a7a69040e77aa51296(size=30.8 K), total size for store is 61.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:26,801 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:26,801 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., storeName=960784f3d73443a7fc815210ff1818b4/A, priority=12, startTime=1732187666297; duration=0sec 2024-11-21T11:14:26,801 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:26,801 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 960784f3d73443a7fc815210ff1818b4:A 2024-11-21T11:14:26,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742011_1187 (size=39749) 2024-11-21T11:14:26,804 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=227, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/e8d36b01d135480897fc379a13e8b6c3 2024-11-21T11:14:26,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/64c0c83f0f2c4551a9f0369e289ba7a4 is 50, key is test_row_0/B:col10/1732187666723/Put/seqid=0 2024-11-21T11:14:26,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742012_1188 (size=12151) 2024-11-21T11:14:26,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/64c0c83f0f2c4551a9f0369e289ba7a4 2024-11-21T11:14:26,830 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/bb262f01070c4c268b052b613972475d as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/bb262f01070c4c268b052b613972475d 2024-11-21T11:14:26,837 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 960784f3d73443a7fc815210ff1818b4/C of 960784f3d73443a7fc815210ff1818b4 into bb262f01070c4c268b052b613972475d(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:26,837 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:26,837 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., storeName=960784f3d73443a7fc815210ff1818b4/C, priority=12, startTime=1732187666298; duration=0sec 2024-11-21T11:14:26,837 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:26,837 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 960784f3d73443a7fc815210ff1818b4:C 2024-11-21T11:14:26,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/449f62f51f3f448db9e2c1d076e7cde7 is 50, key is test_row_0/C:col10/1732187666723/Put/seqid=0 2024-11-21T11:14:26,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742013_1189 (size=12151) 2024-11-21T11:14:26,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187726849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:26,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:26,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:26,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:26,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187726851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:26,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187726851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:26,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:26,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:26,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:26,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:26,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:26,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:26,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187726965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:26,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:26,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:26,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:26,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187726976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:26,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:26,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:26,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:26,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187726976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:26,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:26,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:26,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T11:14:27,056 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:14:27,057 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:14:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-21T11:14:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T11:14:27,059 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:14:27,060 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:14:27,060 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:14:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T11:14:27,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187727186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:27,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:27,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187727186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:27,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:27,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:27,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:27,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:27,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187727186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:27,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T11:14:27,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:27,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:27,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:27,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:27,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:27,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:27,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/449f62f51f3f448db9e2c1d076e7cde7 2024-11-21T11:14:27,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/e8d36b01d135480897fc379a13e8b6c3 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/e8d36b01d135480897fc379a13e8b6c3 2024-11-21T11:14:27,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/e8d36b01d135480897fc379a13e8b6c3, entries=200, sequenceid=227, filesize=38.8 K 2024-11-21T11:14:27,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/64c0c83f0f2c4551a9f0369e289ba7a4 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/64c0c83f0f2c4551a9f0369e289ba7a4 2024-11-21T11:14:27,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/64c0c83f0f2c4551a9f0369e289ba7a4, entries=150, sequenceid=227, filesize=11.9 K 2024-11-21T11:14:27,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/449f62f51f3f448db9e2c1d076e7cde7 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/449f62f51f3f448db9e2c1d076e7cde7 2024-11-21T11:14:27,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/449f62f51f3f448db9e2c1d076e7cde7, entries=150, sequenceid=227, filesize=11.9 K 2024-11-21T11:14:27,280 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=140.89 KB/144270 for 960784f3d73443a7fc815210ff1818b4 in 556ms, sequenceid=227, compaction requested=true 2024-11-21T11:14:27,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:27,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 960784f3d73443a7fc815210ff1818b4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:14:27,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:27,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 960784f3d73443a7fc815210ff1818b4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:14:27,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:27,281 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:14:27,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 960784f3d73443a7fc815210ff1818b4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:14:27,281 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:14:27,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:14:27,282 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:14:27,282 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:14:27,282 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 960784f3d73443a7fc815210ff1818b4/A is initiating minor compaction (all files) 2024-11-21T11:14:27,282 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 960784f3d73443a7fc815210ff1818b4/B is initiating minor compaction (all files) 2024-11-21T11:14:27,282 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 960784f3d73443a7fc815210ff1818b4/A in TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:27,282 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 960784f3d73443a7fc815210ff1818b4/B in TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:27,282 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/e822c0660da149a7a69040e77aa51296, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/29b8abe74cb44a61b4939dd5a63101e8, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/e8d36b01d135480897fc379a13e8b6c3] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp, totalSize=100.0 K 2024-11-21T11:14:27,282 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:27,282 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/aa3c24978c0547a88c36825bb4691b1d, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/94c15e07870d41d885d7e3892f1bf62c, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/64c0c83f0f2c4551a9f0369e289ba7a4] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp, totalSize=36.0 K 2024-11-21T11:14:27,282 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. files: [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/e822c0660da149a7a69040e77aa51296, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/29b8abe74cb44a61b4939dd5a63101e8, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/e8d36b01d135480897fc379a13e8b6c3] 2024-11-21T11:14:27,283 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting e822c0660da149a7a69040e77aa51296, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732187665005 2024-11-21T11:14:27,283 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting aa3c24978c0547a88c36825bb4691b1d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732187665005 2024-11-21T11:14:27,283 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 94c15e07870d41d885d7e3892f1bf62c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732187665361 2024-11-21T11:14:27,284 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29b8abe74cb44a61b4939dd5a63101e8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732187665361 2024-11-21T11:14:27,285 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 64c0c83f0f2c4551a9f0369e289ba7a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732187666569 2024-11-21T11:14:27,285 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8d36b01d135480897fc379a13e8b6c3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732187666569 2024-11-21T11:14:27,307 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 960784f3d73443a7fc815210ff1818b4#B#compaction#158 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:27,308 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/81022537f5984a0d96bf5d9a93b53c1c is 50, key is test_row_0/B:col10/1732187666723/Put/seqid=0 2024-11-21T11:14:27,309 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:27,313 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411211257e19ff2494fe5a0b4a50c03931bea_960784f3d73443a7fc815210ff1818b4 store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:27,316 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411211257e19ff2494fe5a0b4a50c03931bea_960784f3d73443a7fc815210ff1818b4, store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:27,316 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411211257e19ff2494fe5a0b4a50c03931bea_960784f3d73443a7fc815210ff1818b4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:27,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742014_1190 (size=12697) 2024-11-21T11:14:27,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742015_1191 (size=4469) 2024-11-21T11:14:27,355 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 960784f3d73443a7fc815210ff1818b4#A#compaction#159 average throughput is 0.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:27,356 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/23fa475952e741858588ec20a4c80956 is 175, key is test_row_0/A:col10/1732187666723/Put/seqid=0 2024-11-21T11:14:27,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742016_1192 (size=31651) 2024-11-21T11:14:27,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T11:14:27,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:27,370 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 960784f3d73443a7fc815210ff1818b4 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-21T11:14:27,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=A 2024-11-21T11:14:27,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:14:27,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=B 2024-11-21T11:14:27,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:14:27,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=C 2024-11-21T11:14:27,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:14:27,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T11:14:27,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e486c3499f4e40af81f4cc292e821f0f_960784f3d73443a7fc815210ff1818b4 is 50, key is test_row_0/A:col10/1732187666847/Put/seqid=0 2024-11-21T11:14:27,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742017_1193 (size=12404) 2024-11-21T11:14:27,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:27,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:27,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:27,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187727514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:27,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:27,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:27,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:27,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187727515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:27,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:27,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:27,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:27,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:27,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:27,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732187727515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:27,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:27,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187727516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:27,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:27,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:27,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:27,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:27,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187727516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:27,527 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_0' in 960784f3d73443a7fc815210ff1818b4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8252 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:14:27,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:27,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:27,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:27,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:27,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:27,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:27,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:27,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187727625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:27,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:27,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187727625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:27,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:27,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:27,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:27,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:27,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:27,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:27,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187727636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:27,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:27,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:27,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187727636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:27,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:27,638 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:27,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:27,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:27,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T11:14:27,749 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/81022537f5984a0d96bf5d9a93b53c1c as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/81022537f5984a0d96bf5d9a93b53c1c 2024-11-21T11:14:27,755 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 960784f3d73443a7fc815210ff1818b4/B of 960784f3d73443a7fc815210ff1818b4 into 81022537f5984a0d96bf5d9a93b53c1c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:27,756 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:27,756 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., storeName=960784f3d73443a7fc815210ff1818b4/B, priority=13, startTime=1732187667281; duration=0sec 2024-11-21T11:14:27,756 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:14:27,756 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 960784f3d73443a7fc815210ff1818b4:B 2024-11-21T11:14:27,756 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:14:27,757 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:14:27,758 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 960784f3d73443a7fc815210ff1818b4/C is initiating minor compaction (all files) 2024-11-21T11:14:27,758 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 960784f3d73443a7fc815210ff1818b4/C in TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:27,758 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/bb262f01070c4c268b052b613972475d, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/02b3c9440844466a800778e818203568, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/449f62f51f3f448db9e2c1d076e7cde7] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp, totalSize=36.0 K 2024-11-21T11:14:27,759 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting bb262f01070c4c268b052b613972475d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732187665005 2024-11-21T11:14:27,759 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 02b3c9440844466a800778e818203568, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732187665361 2024-11-21T11:14:27,759 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 449f62f51f3f448db9e2c1d076e7cde7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732187666569 2024-11-21T11:14:27,768 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 960784f3d73443a7fc815210ff1818b4#C#compaction#161 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:27,770 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/bea260807f45457394bb350def2448e9 is 50, key is test_row_0/C:col10/1732187666723/Put/seqid=0 2024-11-21T11:14:27,776 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/23fa475952e741858588ec20a4c80956 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/23fa475952e741858588ec20a4c80956 2024-11-21T11:14:27,781 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 960784f3d73443a7fc815210ff1818b4/A of 960784f3d73443a7fc815210ff1818b4 into 23fa475952e741858588ec20a4c80956(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:27,782 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:27,782 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., storeName=960784f3d73443a7fc815210ff1818b4/A, priority=13, startTime=1732187667281; duration=0sec 2024-11-21T11:14:27,782 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:27,782 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 960784f3d73443a7fc815210ff1818b4:A 2024-11-21T11:14:27,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:27,798 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e486c3499f4e40af81f4cc292e821f0f_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e486c3499f4e40af81f4cc292e821f0f_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:27,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/92c46b58e47f43619f10a7474a046d1d, store: [table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:27,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/92c46b58e47f43619f10a7474a046d1d is 175, key is test_row_0/A:col10/1732187666847/Put/seqid=0 2024-11-21T11:14:27,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742018_1194 (size=12663) 2024-11-21T11:14:27,828 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/bea260807f45457394bb350def2448e9 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/bea260807f45457394bb350def2448e9 2024-11-21T11:14:27,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742019_1195 (size=31205) 2024-11-21T11:14:27,832 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=265, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/92c46b58e47f43619f10a7474a046d1d 2024-11-21T11:14:27,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:27,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187727836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:27,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:27,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:27,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187727837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:27,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:27,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:27,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:27,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:27,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:27,840 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 960784f3d73443a7fc815210ff1818b4/C of 960784f3d73443a7fc815210ff1818b4 into bea260807f45457394bb350def2448e9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:27,840 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:27,840 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., storeName=960784f3d73443a7fc815210ff1818b4/C, priority=13, startTime=1732187667281; duration=0sec 2024-11-21T11:14:27,840 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:27,840 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 960784f3d73443a7fc815210ff1818b4:C 2024-11-21T11:14:27,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:27,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187727845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:27,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:27,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187727846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:27,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:27,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:27,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:27,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:27,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:27,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:27,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/4a69a46211c7434c95165dff6028b02c is 50, key is test_row_0/B:col10/1732187666847/Put/seqid=0 2024-11-21T11:14:27,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742020_1196 (size=12251) 2024-11-21T11:14:27,876 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/4a69a46211c7434c95165dff6028b02c 2024-11-21T11:14:27,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/358e7580168442d4ab4f1eec0430a0bd is 50, key is test_row_0/C:col10/1732187666847/Put/seqid=0 2024-11-21T11:14:27,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742021_1197 (size=12251) 2024-11-21T11:14:27,902 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/358e7580168442d4ab4f1eec0430a0bd 2024-11-21T11:14:27,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/92c46b58e47f43619f10a7474a046d1d as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/92c46b58e47f43619f10a7474a046d1d 2024-11-21T11:14:27,921 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/92c46b58e47f43619f10a7474a046d1d, entries=150, sequenceid=265, filesize=30.5 K 2024-11-21T11:14:27,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-21T11:14:27,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/4a69a46211c7434c95165dff6028b02c as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/4a69a46211c7434c95165dff6028b02c 2024-11-21T11:14:27,931 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/4a69a46211c7434c95165dff6028b02c, entries=150, sequenceid=265, filesize=12.0 K 2024-11-21T11:14:27,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/358e7580168442d4ab4f1eec0430a0bd as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/358e7580168442d4ab4f1eec0430a0bd 2024-11-21T11:14:27,936 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/358e7580168442d4ab4f1eec0430a0bd, entries=150, sequenceid=265, filesize=12.0 K 2024-11-21T11:14:27,937 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 960784f3d73443a7fc815210ff1818b4 in 567ms, sequenceid=265, compaction requested=false 2024-11-21T11:14:27,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:27,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:27,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-21T11:14:27,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-21T11:14:27,943 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-21T11:14:27,943 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 881 msec 2024-11-21T11:14:27,946 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 887 msec 2024-11-21T11:14:28,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:28,165 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 960784f3d73443a7fc815210ff1818b4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T11:14:28,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=A 2024-11-21T11:14:28,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:28,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=B 2024-11-21T11:14:28,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:28,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=C 2024-11-21T11:14:28,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:28,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f9cc7af5667044dc81efe468b49b7224_960784f3d73443a7fc815210ff1818b4 is 50, key is test_row_0/A:col10/1732187668146/Put/seqid=0 2024-11-21T11:14:28,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T11:14:28,196 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:14:28,196 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:14:28,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-21T11:14:28,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T11:14:28,200 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:14:28,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742022_1198 (size=14994) 2024-11-21T11:14:28,202 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:14:28,202 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:14:28,209 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:28,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732187668191, max timestamp=1732187668211Num uniques 859; 2024-11-21T11:14:28,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T11:14:28,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732187668191, max timestamp=1732187668211Num uniques 859; 2024-11-21T11:14:28,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T11:14:28,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732187668191, max timestamp=1732187668211Num uniques 859; 2024-11-21T11:14:28,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T11:14:28,219 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f9cc7af5667044dc81efe468b49b7224_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f9cc7af5667044dc81efe468b49b7224_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:28,220 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/defa2e0ea44f4f89b5f0d26622a36d5c, store: [table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:28,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/defa2e0ea44f4f89b5f0d26622a36d5c is 175, key is test_row_0/A:col10/1732187668146/Put/seqid=0 2024-11-21T11:14:28,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742023_1199 (size=39949) 2024-11-21T11:14:28,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:28,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187728249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:28,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:28,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:28,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187728251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:28,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:28,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:28,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:28,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:28,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:28,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:28,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187728258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:28,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:28,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187728260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:28,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:28,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:28,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:28,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:28,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:28,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:28,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T11:14:28,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-21T11:14:28,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:28,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:28,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:28,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:28,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:28,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:28,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187728366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:28,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:28,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:28,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:28,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187728366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:28,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:28,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:28,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:28,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:28,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187728375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:28,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:28,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:28,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:28,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:28,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187728376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:28,385 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:28,385 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:28,385 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:28,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-21T11:14:28,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:28,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:28,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:28,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:28,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T11:14:28,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:28,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187728586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:28,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:28,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:28,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187728586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:28,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:28,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:28,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:28,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:28,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:28,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:28,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187728596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:28,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:28,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:28,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:28,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:28,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187728596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:28,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:28,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:28,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:28,626 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=278, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/defa2e0ea44f4f89b5f0d26622a36d5c 2024-11-21T11:14:28,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/5ec4ea9d88934347a31074279ae03673 is 50, key is test_row_0/B:col10/1732187668146/Put/seqid=0 2024-11-21T11:14:28,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-21T11:14:28,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:28,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:28,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:28,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:28,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:28,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742024_1200 (size=12301) 2024-11-21T11:14:28,692 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/5ec4ea9d88934347a31074279ae03673 2024-11-21T11:14:28,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/836eea3068f1414c847a117940ced7e4 is 50, key is test_row_0/C:col10/1732187668146/Put/seqid=0 2024-11-21T11:14:28,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742025_1201 (size=12301) 2024-11-21T11:14:28,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/836eea3068f1414c847a117940ced7e4 2024-11-21T11:14:28,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/defa2e0ea44f4f89b5f0d26622a36d5c as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/defa2e0ea44f4f89b5f0d26622a36d5c 2024-11-21T11:14:28,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/defa2e0ea44f4f89b5f0d26622a36d5c, entries=200, sequenceid=278, filesize=39.0 K 2024-11-21T11:14:28,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/5ec4ea9d88934347a31074279ae03673 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/5ec4ea9d88934347a31074279ae03673 2024-11-21T11:14:28,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/5ec4ea9d88934347a31074279ae03673, entries=150, sequenceid=278, filesize=12.0 K 2024-11-21T11:14:28,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/836eea3068f1414c847a117940ced7e4 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/836eea3068f1414c847a117940ced7e4 2024-11-21T11:14:28,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/836eea3068f1414c847a117940ced7e4, entries=150, sequenceid=278, filesize=12.0 K 2024-11-21T11:14:28,769 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 960784f3d73443a7fc815210ff1818b4 in 621ms, sequenceid=278, compaction requested=true 2024-11-21T11:14:28,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:28,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 960784f3d73443a7fc815210ff1818b4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:14:28,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:28,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 960784f3d73443a7fc815210ff1818b4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:14:28,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:14:28,769 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:14:28,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 960784f3d73443a7fc815210ff1818b4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:14:28,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T11:14:28,769 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:14:28,770 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102805 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:14:28,770 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 960784f3d73443a7fc815210ff1818b4/A is initiating minor compaction (all files) 2024-11-21T11:14:28,770 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 960784f3d73443a7fc815210ff1818b4/A in TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:28,771 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/23fa475952e741858588ec20a4c80956, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/92c46b58e47f43619f10a7474a046d1d, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/defa2e0ea44f4f89b5f0d26622a36d5c] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp, totalSize=100.4 K 2024-11-21T11:14:28,771 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:28,771 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. files: [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/23fa475952e741858588ec20a4c80956, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/92c46b58e47f43619f10a7474a046d1d, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/defa2e0ea44f4f89b5f0d26622a36d5c] 2024-11-21T11:14:28,771 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 23fa475952e741858588ec20a4c80956, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732187666569 2024-11-21T11:14:28,772 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37249 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:14:28,772 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 960784f3d73443a7fc815210ff1818b4/B is initiating minor compaction (all files) 2024-11-21T11:14:28,772 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 960784f3d73443a7fc815210ff1818b4/B in TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:28,772 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/81022537f5984a0d96bf5d9a93b53c1c, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/4a69a46211c7434c95165dff6028b02c, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/5ec4ea9d88934347a31074279ae03673] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp, totalSize=36.4 K 2024-11-21T11:14:28,773 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 92c46b58e47f43619f10a7474a046d1d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732187666818 2024-11-21T11:14:28,773 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81022537f5984a0d96bf5d9a93b53c1c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732187666569 2024-11-21T11:14:28,773 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a69a46211c7434c95165dff6028b02c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732187666818 2024-11-21T11:14:28,774 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting defa2e0ea44f4f89b5f0d26622a36d5c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732187667512 2024-11-21T11:14:28,775 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ec4ea9d88934347a31074279ae03673, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732187667512 2024-11-21T11:14:28,797 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:28,811 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 960784f3d73443a7fc815210ff1818b4#B#compaction#168 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:28,812 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/301b79fe32c34f8d9b4d9adfac0c6807 is 50, key is test_row_0/B:col10/1732187668146/Put/seqid=0 2024-11-21T11:14:28,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-21T11:14:28,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:28,821 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112174e7c42a00d8403195d5a03b2fd37c37_960784f3d73443a7fc815210ff1818b4 store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:28,821 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 960784f3d73443a7fc815210ff1818b4 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-21T11:14:28,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=A 2024-11-21T11:14:28,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:14:28,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=B 2024-11-21T11:14:28,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:14:28,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=C 2024-11-21T11:14:28,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:14:28,823 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112174e7c42a00d8403195d5a03b2fd37c37_960784f3d73443a7fc815210ff1818b4, store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:28,823 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112174e7c42a00d8403195d5a03b2fd37c37_960784f3d73443a7fc815210ff1818b4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:28,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T11:14:28,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742027_1203 (size=4469) 2024-11-21T11:14:28,889 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 960784f3d73443a7fc815210ff1818b4#A#compaction#167 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:28,890 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/4a45788f1e334f6ebb83ec8a1c59a18e is 175, key is test_row_0/A:col10/1732187668146/Put/seqid=0 2024-11-21T11:14:28,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:28,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742026_1202 (size=12949) 2024-11-21T11:14:28,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112152d29e7c17634a68a1856a9efbe237cd_960784f3d73443a7fc815210ff1818b4 is 50, key is test_row_0/A:col10/1732187668248/Put/seqid=0 2024-11-21T11:14:28,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:28,914 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/301b79fe32c34f8d9b4d9adfac0c6807 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/301b79fe32c34f8d9b4d9adfac0c6807 2024-11-21T11:14:28,921 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 960784f3d73443a7fc815210ff1818b4/B of 960784f3d73443a7fc815210ff1818b4 into 301b79fe32c34f8d9b4d9adfac0c6807(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:28,921 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:28,921 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., storeName=960784f3d73443a7fc815210ff1818b4/B, priority=13, startTime=1732187668769; duration=0sec 2024-11-21T11:14:28,922 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:14:28,922 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 960784f3d73443a7fc815210ff1818b4:B 2024-11-21T11:14:28,924 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:14:28,925 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37215 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:14:28,925 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 960784f3d73443a7fc815210ff1818b4/C is initiating minor compaction (all files) 2024-11-21T11:14:28,926 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 960784f3d73443a7fc815210ff1818b4/C in TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:28,926 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/bea260807f45457394bb350def2448e9, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/358e7580168442d4ab4f1eec0430a0bd, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/836eea3068f1414c847a117940ced7e4] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp, totalSize=36.3 K 2024-11-21T11:14:28,926 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting bea260807f45457394bb350def2448e9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732187666569 2024-11-21T11:14:28,927 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 358e7580168442d4ab4f1eec0430a0bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732187666818 2024-11-21T11:14:28,927 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 836eea3068f1414c847a117940ced7e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732187667512 2024-11-21T11:14:28,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742028_1204 (size=31903) 2024-11-21T11:14:28,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:28,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187728922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:28,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:28,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187728923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:28,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:28,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187728927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:28,948 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/4a45788f1e334f6ebb83ec8a1c59a18e as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/4a45788f1e334f6ebb83ec8a1c59a18e 2024-11-21T11:14:28,955 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 960784f3d73443a7fc815210ff1818b4/A of 960784f3d73443a7fc815210ff1818b4 into 4a45788f1e334f6ebb83ec8a1c59a18e(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:28,955 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:28,955 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., storeName=960784f3d73443a7fc815210ff1818b4/A, priority=13, startTime=1732187668769; duration=0sec 2024-11-21T11:14:28,955 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:28,955 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 960784f3d73443a7fc815210ff1818b4:A 2024-11-21T11:14:28,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:28,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187728942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:28,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:28,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:28,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:28,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742029_1205 (size=12454) 2024-11-21T11:14:28,983 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 960784f3d73443a7fc815210ff1818b4#C#compaction#170 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:28,984 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/d1da2b83f81741418952eab3fdd0aa05 is 50, key is test_row_0/C:col10/1732187668146/Put/seqid=0 2024-11-21T11:14:28,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742030_1206 (size=12915) 2024-11-21T11:14:29,002 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/d1da2b83f81741418952eab3fdd0aa05 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/d1da2b83f81741418952eab3fdd0aa05 2024-11-21T11:14:29,022 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 960784f3d73443a7fc815210ff1818b4/C of 960784f3d73443a7fc815210ff1818b4 into d1da2b83f81741418952eab3fdd0aa05(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:29,023 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:29,023 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., storeName=960784f3d73443a7fc815210ff1818b4/C, priority=13, startTime=1732187668769; duration=0sec 2024-11-21T11:14:29,023 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:29,023 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 960784f3d73443a7fc815210ff1818b4:C 2024-11-21T11:14:29,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187729046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:29,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:29,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:29,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:29,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187729046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:29,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:29,057 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:29,057 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:29,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187729046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:29,057 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:29,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:29,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:29,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:29,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:29,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187729065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:29,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:29,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:29,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:29,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187729266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:29,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187729266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:29,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187729266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:29,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:29,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:29,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:29,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:29,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:29,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:29,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:29,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187729286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:29,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:29,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:29,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T11:14:29,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:29,374 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112152d29e7c17634a68a1856a9efbe237cd_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112152d29e7c17634a68a1856a9efbe237cd_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:29,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/e3595e8d0c29427086a82259eb52e26a, store: [table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:29,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/e3595e8d0c29427086a82259eb52e26a is 175, key is test_row_0/A:col10/1732187668248/Put/seqid=0 2024-11-21T11:14:29,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742031_1207 (size=31255) 2024-11-21T11:14:29,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:29,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187729575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:29,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:29,580 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:29,580 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:29,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187729575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:29,580 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:29,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:29,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187729575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:29,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:29,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:29,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:29,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:29,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:29,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:29,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187729595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:29,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:29,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:29,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:29,793 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/e3595e8d0c29427086a82259eb52e26a 2024-11-21T11:14:29,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/4ea154aa87a4431295edcff73bdc3ffa is 50, key is test_row_0/B:col10/1732187668248/Put/seqid=0 2024-11-21T11:14:29,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742032_1208 (size=12301) 2024-11-21T11:14:29,807 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/4ea154aa87a4431295edcff73bdc3ffa 2024-11-21T11:14:29,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/b2910e0267334c0ea37be0d9ff0f8984 is 50, key is test_row_0/C:col10/1732187668248/Put/seqid=0 2024-11-21T11:14:29,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742033_1209 (size=12301) 2024-11-21T11:14:30,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:30,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187730085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:30,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:30,087 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:30,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187730085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:30,087 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:30,087 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:30,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:30,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:30,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:30,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:30,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187730095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:30,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:30,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:30,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:30,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:30,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187730106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:30,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:30,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:30,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:30,223 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/b2910e0267334c0ea37be0d9ff0f8984 2024-11-21T11:14:30,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/e3595e8d0c29427086a82259eb52e26a as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/e3595e8d0c29427086a82259eb52e26a 2024-11-21T11:14:30,236 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/e3595e8d0c29427086a82259eb52e26a, entries=150, sequenceid=313, filesize=30.5 K 2024-11-21T11:14:30,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/4ea154aa87a4431295edcff73bdc3ffa as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/4ea154aa87a4431295edcff73bdc3ffa 2024-11-21T11:14:30,242 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/4ea154aa87a4431295edcff73bdc3ffa, entries=150, sequenceid=313, filesize=12.0 K 2024-11-21T11:14:30,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/b2910e0267334c0ea37be0d9ff0f8984 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/b2910e0267334c0ea37be0d9ff0f8984 2024-11-21T11:14:30,248 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/b2910e0267334c0ea37be0d9ff0f8984, entries=150, sequenceid=313, filesize=12.0 K 2024-11-21T11:14:30,250 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 960784f3d73443a7fc815210ff1818b4 in 1429ms, sequenceid=313, compaction requested=false 2024-11-21T11:14:30,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:30,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:30,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-21T11:14:30,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-21T11:14:30,254 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-21T11:14:30,254 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0500 sec 2024-11-21T11:14:30,256 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.0580 sec 2024-11-21T11:14:30,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T11:14:30,346 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:14:30,347 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:14:30,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-21T11:14:30,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T11:14:30,350 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:14:30,351 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:14:30,351 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:14:30,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T11:14:30,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T11:14:30,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:30,505 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 960784f3d73443a7fc815210ff1818b4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T11:14:30,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=A 2024-11-21T11:14:30,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:30,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=B 2024-11-21T11:14:30,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:30,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=C 2024-11-21T11:14:30,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:30,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411210c3d116136ed47559170e389e924e86f_960784f3d73443a7fc815210ff1818b4 is 50, key is test_row_0/A:col10/1732187668922/Put/seqid=0 2024-11-21T11:14:30,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742034_1210 (size=12454) 2024-11-21T11:14:30,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T11:14:30,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:30,970 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411210c3d116136ed47559170e389e924e86f_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411210c3d116136ed47559170e389e924e86f_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:30,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/d28f73bf45044b89a6174f007c701cfb, store: [table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:30,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/d28f73bf45044b89a6174f007c701cfb is 175, key is test_row_0/A:col10/1732187668922/Put/seqid=0 2024-11-21T11:14:30,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T11:14:30,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742035_1211 (size=31255) 2024-11-21T11:14:30,986 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=326, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/d28f73bf45044b89a6174f007c701cfb 2024-11-21T11:14:31,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/cd9ff6588ca244a2bc6616f9b5cd89aa is 50, key is test_row_0/B:col10/1732187668922/Put/seqid=0 2024-11-21T11:14:31,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742036_1212 (size=12301) 2024-11-21T11:14:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:31,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:31,147 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732187671122, max timestamp=1732187671144Num uniques 859; 2024-11-21T11:14:31,147 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732187671122, max timestamp=1732187671144Num uniques 859; 2024-11-21T11:14:31,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T11:14:31,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732187671122, max timestamp=1732187671144Num uniques 859; 2024-11-21T11:14:31,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T11:14:31,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T11:14:31,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:31,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187731176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:31,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:31,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187731185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:31,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:31,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:31,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:31,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187731187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:31,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:31,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187731188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:31,192 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,192 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,192 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:31,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:31,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:31,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187731295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:31,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:31,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187731295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:31,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:31,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:31,298 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,298 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,298 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:31,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187731295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:31,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:31,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:31,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187731295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:31,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:31,437 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/cd9ff6588ca244a2bc6616f9b5cd89aa 2024-11-21T11:14:31,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/8879f3f0ca54427e8a4f84e8680dda16 is 50, key is test_row_0/C:col10/1732187668922/Put/seqid=0 2024-11-21T11:14:31,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742037_1213 (size=12301) 2024-11-21T11:14:31,452 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/8879f3f0ca54427e8a4f84e8680dda16 2024-11-21T11:14:31,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/d28f73bf45044b89a6174f007c701cfb as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/d28f73bf45044b89a6174f007c701cfb 2024-11-21T11:14:31,466 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/d28f73bf45044b89a6174f007c701cfb, entries=150, sequenceid=326, filesize=30.5 K 2024-11-21T11:14:31,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/cd9ff6588ca244a2bc6616f9b5cd89aa as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/cd9ff6588ca244a2bc6616f9b5cd89aa 2024-11-21T11:14:31,475 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/cd9ff6588ca244a2bc6616f9b5cd89aa, entries=150, sequenceid=326, filesize=12.0 K 2024-11-21T11:14:31,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/8879f3f0ca54427e8a4f84e8680dda16 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/8879f3f0ca54427e8a4f84e8680dda16 2024-11-21T11:14:31,480 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/8879f3f0ca54427e8a4f84e8680dda16, entries=150, sequenceid=326, filesize=12.0 K 2024-11-21T11:14:31,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T11:14:31,485 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 960784f3d73443a7fc815210ff1818b4 in 981ms, sequenceid=326, compaction requested=true 2024-11-21T11:14:31,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:31,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:31,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-21T11:14:31,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-21T11:14:31,488 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-21T11:14:31,488 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1360 sec 2024-11-21T11:14:31,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.1420 sec 2024-11-21T11:14:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:31,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 960784f3d73443a7fc815210ff1818b4 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-11-21T11:14:31,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=A 2024-11-21T11:14:31,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:14:31,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=B 2024-11-21T11:14:31,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:14:31,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=C 2024-11-21T11:14:31,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:14:31,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187731517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:31,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:31,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187731521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:31,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:31,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187731523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:31,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:31,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187731523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:31,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:31,529 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,529 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,529 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:31,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121fd755ec36f3344f39e45beece4d62c00_960784f3d73443a7fc815210ff1818b4 is 50, key is test_row_0/A:col10/1732187671144/Put/seqid=0 2024-11-21T11:14:31,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742038_1214 (size=14994) 2024-11-21T11:14:31,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187731636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:31,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:31,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187731636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:31,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:31,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:31,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187731636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:31,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187731636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:31,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:31,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:31,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:31,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187731846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:31,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:31,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187731846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:31,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:31,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:31,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187731846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:31,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:31,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:31,848 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,848 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187731846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:31,848 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:31,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:31,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:31,946 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:31,958 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121fd755ec36f3344f39e45beece4d62c00_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121fd755ec36f3344f39e45beece4d62c00_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:31,960 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/9d6617d39ede47d3b17b39d3e336b12b, store: [table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:31,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/9d6617d39ede47d3b17b39d3e336b12b is 175, key is test_row_0/A:col10/1732187671144/Put/seqid=0 2024-11-21T11:14:31,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742039_1215 (size=39949) 2024-11-21T11:14:32,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:32,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187732156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:32,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:32,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:32,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:32,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:32,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187732156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:32,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:32,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:32,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:32,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:32,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187732156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:32,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:32,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:32,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:32,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:32,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187732156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:32,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:32,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:32,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:32,370 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=363, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/9d6617d39ede47d3b17b39d3e336b12b 2024-11-21T11:14:32,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/912ecd0ab20f44e2a412847e8ff6d069 is 50, key is test_row_0/B:col10/1732187671144/Put/seqid=0 2024-11-21T11:14:32,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742040_1216 (size=12301) 2024-11-21T11:14:32,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T11:14:32,496 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:14:32,504 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:14:32,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-21T11:14:32,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T11:14:32,507 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:14:32,508 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:14:32,508 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:14:32,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T11:14:32,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-21T11:14:32,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:32,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:32,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:32,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:32,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:32,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:32,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:32,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187732676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:32,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:32,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:32,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:32,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:32,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187732676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:32,682 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:32,682 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:32,682 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:32,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:32,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187732676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:32,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:32,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:32,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:32,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:32,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187732677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:32,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:32,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:32,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:32,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/912ecd0ab20f44e2a412847e8ff6d069 2024-11-21T11:14:32,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/e635d8417b0741499484510cebff3f9a is 50, key is test_row_0/C:col10/1732187671144/Put/seqid=0 2024-11-21T11:14:32,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742041_1217 (size=12301) 2024-11-21T11:14:32,807 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/e635d8417b0741499484510cebff3f9a 2024-11-21T11:14:32,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/9d6617d39ede47d3b17b39d3e336b12b as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/9d6617d39ede47d3b17b39d3e336b12b 2024-11-21T11:14:32,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-21T11:14:32,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:32,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:32,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:32,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:32,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:32,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:32,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/9d6617d39ede47d3b17b39d3e336b12b, entries=200, sequenceid=363, filesize=39.0 K 2024-11-21T11:14:32,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/912ecd0ab20f44e2a412847e8ff6d069 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/912ecd0ab20f44e2a412847e8ff6d069 2024-11-21T11:14:32,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/912ecd0ab20f44e2a412847e8ff6d069, entries=150, sequenceid=363, filesize=12.0 K 2024-11-21T11:14:32,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/e635d8417b0741499484510cebff3f9a as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/e635d8417b0741499484510cebff3f9a 2024-11-21T11:14:32,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T11:14:32,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/e635d8417b0741499484510cebff3f9a, entries=150, sequenceid=363, filesize=12.0 K 2024-11-21T11:14:32,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=40.25 KB/41220 for 960784f3d73443a7fc815210ff1818b4 in 1319ms, sequenceid=363, compaction requested=true 2024-11-21T11:14:32,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:32,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 960784f3d73443a7fc815210ff1818b4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:14:32,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:32,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 960784f3d73443a7fc815210ff1818b4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:14:32,832 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T11:14:32,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:14:32,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 960784f3d73443a7fc815210ff1818b4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:14:32,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T11:14:32,832 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T11:14:32,833 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134362 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T11:14:32,833 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 960784f3d73443a7fc815210ff1818b4/A is initiating minor compaction (all files) 2024-11-21T11:14:32,833 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 960784f3d73443a7fc815210ff1818b4/A in TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:32,834 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T11:14:32,834 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/4a45788f1e334f6ebb83ec8a1c59a18e, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/e3595e8d0c29427086a82259eb52e26a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/d28f73bf45044b89a6174f007c701cfb, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/9d6617d39ede47d3b17b39d3e336b12b] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp, totalSize=131.2 K 2024-11-21T11:14:32,834 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:32,834 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 960784f3d73443a7fc815210ff1818b4/B is initiating minor compaction (all files) 2024-11-21T11:14:32,834 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. files: [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/4a45788f1e334f6ebb83ec8a1c59a18e, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/e3595e8d0c29427086a82259eb52e26a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/d28f73bf45044b89a6174f007c701cfb, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/9d6617d39ede47d3b17b39d3e336b12b] 2024-11-21T11:14:32,834 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 960784f3d73443a7fc815210ff1818b4/B in TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:32,834 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/301b79fe32c34f8d9b4d9adfac0c6807, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/4ea154aa87a4431295edcff73bdc3ffa, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/cd9ff6588ca244a2bc6616f9b5cd89aa, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/912ecd0ab20f44e2a412847e8ff6d069] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp, totalSize=48.7 K 2024-11-21T11:14:32,834 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a45788f1e334f6ebb83ec8a1c59a18e, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732187667512 2024-11-21T11:14:32,835 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 301b79fe32c34f8d9b4d9adfac0c6807, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732187667512 2024-11-21T11:14:32,837 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3595e8d0c29427086a82259eb52e26a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732187668244 2024-11-21T11:14:32,837 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ea154aa87a4431295edcff73bdc3ffa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732187668244 2024-11-21T11:14:32,837 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting d28f73bf45044b89a6174f007c701cfb, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732187668917 2024-11-21T11:14:32,838 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting cd9ff6588ca244a2bc6616f9b5cd89aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732187668917 2024-11-21T11:14:32,838 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d6617d39ede47d3b17b39d3e336b12b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732187671144 2024-11-21T11:14:32,840 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 912ecd0ab20f44e2a412847e8ff6d069, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732187671144 2024-11-21T11:14:32,859 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:32,869 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 960784f3d73443a7fc815210ff1818b4#B#compaction#180 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:32,869 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/29153c0d2f5f430483f57fb5deef3ecd is 50, key is test_row_0/B:col10/1732187671144/Put/seqid=0 2024-11-21T11:14:32,876 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112166372434d2d242f5ab716b04be5bd12a_960784f3d73443a7fc815210ff1818b4 store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:32,879 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112166372434d2d242f5ab716b04be5bd12a_960784f3d73443a7fc815210ff1818b4, store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:32,879 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112166372434d2d242f5ab716b04be5bd12a_960784f3d73443a7fc815210ff1818b4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:32,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742043_1219 (size=4469) 2024-11-21T11:14:32,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742042_1218 (size=13085) 2024-11-21T11:14:32,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-21T11:14:32,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:32,975 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 960784f3d73443a7fc815210ff1818b4 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-21T11:14:32,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=A 2024-11-21T11:14:32,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:32,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=B 2024-11-21T11:14:32,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:32,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=C 2024-11-21T11:14:32,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:32,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b3c4a964efb341908e5ed4503a4368d1_960784f3d73443a7fc815210ff1818b4 is 50, key is test_row_0/A:col10/1732187671519/Put/seqid=0 2024-11-21T11:14:32,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742044_1220 (size=12454) 2024-11-21T11:14:33,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,031 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b3c4a964efb341908e5ed4503a4368d1_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b3c4a964efb341908e5ed4503a4368d1_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:33,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/1412abb3e13e447b8d01c80c9cf2a887, store: [table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:33,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/1412abb3e13e447b8d01c80c9cf2a887 is 175, key is test_row_0/A:col10/1732187671519/Put/seqid=0 2024-11-21T11:14:33,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742045_1221 (size=31255) 2024-11-21T11:14:33,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T11:14:33,314 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 960784f3d73443a7fc815210ff1818b4#A#compaction#179 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:33,315 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/0400606c8d544027a10e8dc71c7e8fa5 is 175, key is test_row_0/A:col10/1732187671144/Put/seqid=0 2024-11-21T11:14:33,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742046_1222 (size=32039) 2024-11-21T11:14:33,324 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/29153c0d2f5f430483f57fb5deef3ecd as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/29153c0d2f5f430483f57fb5deef3ecd 2024-11-21T11:14:33,329 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 960784f3d73443a7fc815210ff1818b4/B of 960784f3d73443a7fc815210ff1818b4 into 29153c0d2f5f430483f57fb5deef3ecd(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:33,329 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:33,330 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., storeName=960784f3d73443a7fc815210ff1818b4/B, priority=12, startTime=1732187672832; duration=0sec 2024-11-21T11:14:33,330 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:14:33,330 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 960784f3d73443a7fc815210ff1818b4:B 2024-11-21T11:14:33,330 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T11:14:33,331 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T11:14:33,331 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 960784f3d73443a7fc815210ff1818b4/C is initiating minor compaction (all files) 2024-11-21T11:14:33,331 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 960784f3d73443a7fc815210ff1818b4/C in TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:33,331 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/d1da2b83f81741418952eab3fdd0aa05, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/b2910e0267334c0ea37be0d9ff0f8984, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/8879f3f0ca54427e8a4f84e8680dda16, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/e635d8417b0741499484510cebff3f9a] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp, totalSize=48.7 K 2024-11-21T11:14:33,332 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting d1da2b83f81741418952eab3fdd0aa05, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732187667512 2024-11-21T11:14:33,332 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting b2910e0267334c0ea37be0d9ff0f8984, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732187668244 2024-11-21T11:14:33,332 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 8879f3f0ca54427e8a4f84e8680dda16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732187668917 2024-11-21T11:14:33,333 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting e635d8417b0741499484510cebff3f9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732187671144 2024-11-21T11:14:33,343 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 960784f3d73443a7fc815210ff1818b4#C#compaction#182 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:33,344 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/28a6082798a7424ca83419b3b041b330 is 50, key is test_row_0/C:col10/1732187671144/Put/seqid=0 2024-11-21T11:14:33,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742047_1223 (size=13051) 2024-11-21T11:14:33,487 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=372, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/1412abb3e13e447b8d01c80c9cf2a887 2024-11-21T11:14:33,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/7a6525114bb34edebfdd4006a3d64840 is 50, key is test_row_0/B:col10/1732187671519/Put/seqid=0 2024-11-21T11:14:33,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742048_1224 (size=12301) 2024-11-21T11:14:33,505 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/7a6525114bb34edebfdd4006a3d64840 2024-11-21T11:14:33,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/7a34158cf44f4b9fbf3b768811c32a57 is 50, key is test_row_0/C:col10/1732187671519/Put/seqid=0 2024-11-21T11:14:33,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742049_1225 (size=12301) 2024-11-21T11:14:33,517 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/7a34158cf44f4b9fbf3b768811c32a57 2024-11-21T11:14:33,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/1412abb3e13e447b8d01c80c9cf2a887 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/1412abb3e13e447b8d01c80c9cf2a887 2024-11-21T11:14:33,527 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/1412abb3e13e447b8d01c80c9cf2a887, entries=150, sequenceid=372, filesize=30.5 K 2024-11-21T11:14:33,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/7a6525114bb34edebfdd4006a3d64840 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/7a6525114bb34edebfdd4006a3d64840 2024-11-21T11:14:33,536 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/7a6525114bb34edebfdd4006a3d64840, entries=150, sequenceid=372, filesize=12.0 K 2024-11-21T11:14:33,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/7a34158cf44f4b9fbf3b768811c32a57 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/7a34158cf44f4b9fbf3b768811c32a57 2024-11-21T11:14:33,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,543 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/7a34158cf44f4b9fbf3b768811c32a57, entries=150, sequenceid=372, filesize=12.0 K 2024-11-21T11:14:33,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,544 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for 960784f3d73443a7fc815210ff1818b4 in 568ms, sequenceid=372, compaction requested=false 2024-11-21T11:14:33,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:33,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:33,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-21T11:14:33,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-21T11:14:33,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,548 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-21T11:14:33,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,548 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0380 sec 2024-11-21T11:14:33,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,550 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.0440 sec 2024-11-21T11:14:33,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T11:14:33,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,645 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:14:33,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,646 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:14:33,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-21T11:14:33,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T11:14:33,649 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:14:33,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,650 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:14:33,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,650 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:14:33,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,727 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/0400606c8d544027a10e8dc71c7e8fa5 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/0400606c8d544027a10e8dc71c7e8fa5 2024-11-21T11:14:33,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,734 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 960784f3d73443a7fc815210ff1818b4/A of 960784f3d73443a7fc815210ff1818b4 into 0400606c8d544027a10e8dc71c7e8fa5(size=31.3 K), total size for store is 61.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:33,734 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:33,734 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., storeName=960784f3d73443a7fc815210ff1818b4/A, priority=12, startTime=1732187672832; duration=0sec 2024-11-21T11:14:33,734 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:33,734 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 960784f3d73443a7fc815210ff1818b4:A 2024-11-21T11:14:33,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T11:14:33,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,764 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/28a6082798a7424ca83419b3b041b330 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/28a6082798a7424ca83419b3b041b330 2024-11-21T11:14:33,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:33,765 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 960784f3d73443a7fc815210ff1818b4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T11:14:33,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=A 2024-11-21T11:14:33,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:33,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=B 2024-11-21T11:14:33,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:33,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=C 2024-11-21T11:14:33,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:33,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,773 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 960784f3d73443a7fc815210ff1818b4/C of 960784f3d73443a7fc815210ff1818b4 into 28a6082798a7424ca83419b3b041b330(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:33,773 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:33,773 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., storeName=960784f3d73443a7fc815210ff1818b4/C, priority=12, startTime=1732187672832; duration=0sec 2024-11-21T11:14:33,773 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:33,773 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 960784f3d73443a7fc815210ff1818b4:C 2024-11-21T11:14:33,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215290bd3d45904f1297657d2ff9d27b47_960784f3d73443a7fc815210ff1818b4 is 50, key is test_row_0/A:col10/1732187673763/Put/seqid=0 2024-11-21T11:14:33,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-21T11:14:33,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:33,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:33,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:33,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:33,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:33,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:33,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742050_1226 (size=22618) 2024-11-21T11:14:33,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,826 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,831 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215290bd3d45904f1297657d2ff9d27b47_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215290bd3d45904f1297657d2ff9d27b47_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:33,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,833 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/9bbc0b2575ef495ba77667e2442be918, store: [table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:33,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/9bbc0b2575ef495ba77667e2442be918 is 175, key is test_row_0/A:col10/1732187673763/Put/seqid=0 2024-11-21T11:14:33,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732187673818, max timestamp=1732187673844Num uniques 859; 2024-11-21T11:14:33,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T11:14:33,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732187673818, max timestamp=1732187673844Num uniques 859; 2024-11-21T11:14:33,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732187673818, max timestamp=1732187673844Num uniques 859; 2024-11-21T11:14:33,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T11:14:33,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T11:14:33,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742051_1227 (size=66023) 2024-11-21T11:14:33,867 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=384, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/9bbc0b2575ef495ba77667e2442be918 2024-11-21T11:14:33,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/211421efd3cf482d935afd59bcc69089 is 50, key is test_row_0/B:col10/1732187673763/Put/seqid=0 2024-11-21T11:14:33,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:33,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187733887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:33,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742052_1228 (size=12301) 2024-11-21T11:14:33,906 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/211421efd3cf482d935afd59bcc69089 2024-11-21T11:14:33,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:33,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187733896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:33,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:33,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:33,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:33,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:33,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187733897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:33,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:33,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:33,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:33,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187733900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:33,910 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:33,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:33,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:33,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:33,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/d58507e821454058adc559b0758bbff2 is 50, key is test_row_0/C:col10/1732187673763/Put/seqid=0 2024-11-21T11:14:33,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-21T11:14:33,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:33,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:33,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:33,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:33,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:33,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T11:14:33,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742053_1229 (size=12301) 2024-11-21T11:14:33,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:34,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:34,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187734006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:34,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:34,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:34,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187734016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:34,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:34,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187734016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:34,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:34,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:34,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187734016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:34,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:34,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:34,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-21T11:14:34,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:34,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:34,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:34,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:34,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:34,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:34,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:34,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187734226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:34,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:34,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187734226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:34,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:34,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:34,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187734226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:34,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:34,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:34,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187734226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:34,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:34,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:34,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T11:14:34,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-21T11:14:34,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:34,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:34,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:34,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:34,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:34,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:34,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/d58507e821454058adc559b0758bbff2 2024-11-21T11:14:34,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/9bbc0b2575ef495ba77667e2442be918 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/9bbc0b2575ef495ba77667e2442be918 2024-11-21T11:14:34,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/9bbc0b2575ef495ba77667e2442be918, entries=350, sequenceid=384, filesize=64.5 K 2024-11-21T11:14:34,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/211421efd3cf482d935afd59bcc69089 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/211421efd3cf482d935afd59bcc69089 2024-11-21T11:14:34,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/211421efd3cf482d935afd59bcc69089, entries=150, sequenceid=384, filesize=12.0 K 2024-11-21T11:14:34,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/d58507e821454058adc559b0758bbff2 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/d58507e821454058adc559b0758bbff2 2024-11-21T11:14:34,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/d58507e821454058adc559b0758bbff2, entries=150, sequenceid=384, filesize=12.0 K 2024-11-21T11:14:34,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 960784f3d73443a7fc815210ff1818b4 in 621ms, sequenceid=384, compaction requested=true 2024-11-21T11:14:34,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:34,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 960784f3d73443a7fc815210ff1818b4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:14:34,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:14:34,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 960784f3d73443a7fc815210ff1818b4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:14:34,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:14:34,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 960784f3d73443a7fc815210ff1818b4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:14:34,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T11:14:34,386 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:14:34,386 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:14:34,395 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 129317 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:14:34,395 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 960784f3d73443a7fc815210ff1818b4/A is initiating minor compaction (all files) 2024-11-21T11:14:34,395 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 960784f3d73443a7fc815210ff1818b4/A in TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:34,395 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/0400606c8d544027a10e8dc71c7e8fa5, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/1412abb3e13e447b8d01c80c9cf2a887, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/9bbc0b2575ef495ba77667e2442be918] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp, totalSize=126.3 K 2024-11-21T11:14:34,395 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:34,395 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. files: [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/0400606c8d544027a10e8dc71c7e8fa5, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/1412abb3e13e447b8d01c80c9cf2a887, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/9bbc0b2575ef495ba77667e2442be918] 2024-11-21T11:14:34,395 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:14:34,396 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 960784f3d73443a7fc815210ff1818b4/B is initiating minor compaction (all files) 2024-11-21T11:14:34,396 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 960784f3d73443a7fc815210ff1818b4/B in TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:34,396 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/29153c0d2f5f430483f57fb5deef3ecd, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/7a6525114bb34edebfdd4006a3d64840, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/211421efd3cf482d935afd59bcc69089] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp, totalSize=36.8 K 2024-11-21T11:14:34,396 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 0400606c8d544027a10e8dc71c7e8fa5, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732187671144 2024-11-21T11:14:34,396 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29153c0d2f5f430483f57fb5deef3ecd, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732187671144 2024-11-21T11:14:34,397 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 1412abb3e13e447b8d01c80c9cf2a887, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732187671519 2024-11-21T11:14:34,397 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a6525114bb34edebfdd4006a3d64840, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732187671519 2024-11-21T11:14:34,397 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 211421efd3cf482d935afd59bcc69089, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732187673745 2024-11-21T11:14:34,398 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bbc0b2575ef495ba77667e2442be918, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732187673740 2024-11-21T11:14:34,416 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 960784f3d73443a7fc815210ff1818b4#B#compaction#188 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:34,417 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/93e0f09f2cd3444eb1b5619d31045234 is 50, key is test_row_0/B:col10/1732187673763/Put/seqid=0 2024-11-21T11:14:34,420 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:34,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742054_1230 (size=13187) 2024-11-21T11:14:34,439 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411215387941a91434ddb8e3a8ae22c537f65_960784f3d73443a7fc815210ff1818b4 store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:34,442 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411215387941a91434ddb8e3a8ae22c537f65_960784f3d73443a7fc815210ff1818b4, store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:34,442 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215387941a91434ddb8e3a8ae22c537f65_960784f3d73443a7fc815210ff1818b4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:34,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-21T11:14:34,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:34,447 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 960784f3d73443a7fc815210ff1818b4 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-21T11:14:34,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=A 2024-11-21T11:14:34,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:14:34,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=B 2024-11-21T11:14:34,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:14:34,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=C 2024-11-21T11:14:34,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:14:34,452 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/93e0f09f2cd3444eb1b5619d31045234 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/93e0f09f2cd3444eb1b5619d31045234 2024-11-21T11:14:34,464 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 960784f3d73443a7fc815210ff1818b4/B of 960784f3d73443a7fc815210ff1818b4 into 93e0f09f2cd3444eb1b5619d31045234(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:34,464 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:34,464 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., storeName=960784f3d73443a7fc815210ff1818b4/B, priority=13, startTime=1732187674386; duration=0sec 2024-11-21T11:14:34,464 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:14:34,464 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 960784f3d73443a7fc815210ff1818b4:B 2024-11-21T11:14:34,464 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:14:34,467 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:14:34,468 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 960784f3d73443a7fc815210ff1818b4/C is initiating minor compaction (all files) 2024-11-21T11:14:34,468 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 960784f3d73443a7fc815210ff1818b4/C in TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:34,468 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/28a6082798a7424ca83419b3b041b330, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/7a34158cf44f4b9fbf3b768811c32a57, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/d58507e821454058adc559b0758bbff2] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp, totalSize=36.8 K 2024-11-21T11:14:34,468 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28a6082798a7424ca83419b3b041b330, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732187671144 2024-11-21T11:14:34,472 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a34158cf44f4b9fbf3b768811c32a57, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732187671519 2024-11-21T11:14:34,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215182e3c9c24140be849170758c3210da_960784f3d73443a7fc815210ff1818b4 is 50, key is test_row_0/A:col10/1732187673885/Put/seqid=0 2024-11-21T11:14:34,475 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting d58507e821454058adc559b0758bbff2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732187673745 2024-11-21T11:14:34,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742055_1231 (size=4469) 2024-11-21T11:14:34,506 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 960784f3d73443a7fc815210ff1818b4#A#compaction#189 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:34,506 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/270b4a5d0c7046ba9b7d21d7e73a26ef is 175, key is test_row_0/A:col10/1732187673763/Put/seqid=0 2024-11-21T11:14:34,532 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 960784f3d73443a7fc815210ff1818b4#C#compaction#191 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:34,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742056_1232 (size=12454) 2024-11-21T11:14:34,533 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/a8d4e223c30646de8954f6b6f871e78c is 50, key is test_row_0/C:col10/1732187673763/Put/seqid=0 2024-11-21T11:14:34,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:34,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:34,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:34,541 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215182e3c9c24140be849170758c3210da_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215182e3c9c24140be849170758c3210da_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:34,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/17faa47ac9da49148da712df321d3aa2, store: [table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:34,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/17faa47ac9da49148da712df321d3aa2 is 175, key is test_row_0/A:col10/1732187673885/Put/seqid=0 2024-11-21T11:14:34,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742058_1234 (size=13153) 2024-11-21T11:14:34,562 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/a8d4e223c30646de8954f6b6f871e78c as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/a8d4e223c30646de8954f6b6f871e78c 2024-11-21T11:14:34,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742057_1233 (size=32141) 2024-11-21T11:14:34,576 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/270b4a5d0c7046ba9b7d21d7e73a26ef as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/270b4a5d0c7046ba9b7d21d7e73a26ef 2024-11-21T11:14:34,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742059_1235 (size=31255) 2024-11-21T11:14:34,590 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=422, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/17faa47ac9da49148da712df321d3aa2 2024-11-21T11:14:34,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/dc8cce0574174e6481f2120def6abd1f is 50, key is test_row_0/B:col10/1732187673885/Put/seqid=0 2024-11-21T11:14:34,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742060_1236 (size=12301) 2024-11-21T11:14:34,616 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 960784f3d73443a7fc815210ff1818b4/C of 960784f3d73443a7fc815210ff1818b4 into a8d4e223c30646de8954f6b6f871e78c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:34,616 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:34,616 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., storeName=960784f3d73443a7fc815210ff1818b4/C, priority=13, startTime=1732187674386; duration=0sec 2024-11-21T11:14:34,616 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:34,616 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 960784f3d73443a7fc815210ff1818b4:C 2024-11-21T11:14:34,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:34,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187734560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:34,618 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 960784f3d73443a7fc815210ff1818b4/A of 960784f3d73443a7fc815210ff1818b4 into 270b4a5d0c7046ba9b7d21d7e73a26ef(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:34,618 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:34,618 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., storeName=960784f3d73443a7fc815210ff1818b4/A, priority=13, startTime=1732187674386; duration=0sec 2024-11-21T11:14:34,618 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:34,618 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 960784f3d73443a7fc815210ff1818b4:A 2024-11-21T11:14:34,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:34,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:34,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187734617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:34,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:34,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:34,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187734618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:34,628 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,628 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,628 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:34,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:34,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187734618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:34,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:34,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:34,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187734725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:34,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:34,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187734735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:34,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187734736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:34,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187734736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:34,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:34,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:34,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T11:14:34,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:34,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187734935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:34,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:34,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187734945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:34,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:34,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:34,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187734945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:34,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:34,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:34,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187734956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:34,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:34,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:35,008 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/dc8cce0574174e6481f2120def6abd1f 2024-11-21T11:14:35,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/ab295362759646a186cd85b515dd2123 is 50, key is test_row_0/C:col10/1732187673885/Put/seqid=0 2024-11-21T11:14:35,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742061_1237 (size=12301) 2024-11-21T11:14:35,031 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/ab295362759646a186cd85b515dd2123 2024-11-21T11:14:35,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/17faa47ac9da49148da712df321d3aa2 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/17faa47ac9da49148da712df321d3aa2 2024-11-21T11:14:35,045 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/17faa47ac9da49148da712df321d3aa2, entries=150, sequenceid=422, filesize=30.5 K 2024-11-21T11:14:35,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/dc8cce0574174e6481f2120def6abd1f as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/dc8cce0574174e6481f2120def6abd1f 2024-11-21T11:14:35,056 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/dc8cce0574174e6481f2120def6abd1f, entries=150, sequenceid=422, filesize=12.0 K 2024-11-21T11:14:35,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/ab295362759646a186cd85b515dd2123 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/ab295362759646a186cd85b515dd2123 2024-11-21T11:14:35,063 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/ab295362759646a186cd85b515dd2123, entries=150, sequenceid=422, filesize=12.0 K 2024-11-21T11:14:35,064 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=53.67 KB/54960 for 960784f3d73443a7fc815210ff1818b4 in 617ms, sequenceid=422, compaction requested=false 2024-11-21T11:14:35,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:35,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:35,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-21T11:14:35,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-21T11:14:35,067 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-21T11:14:35,067 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4160 sec 2024-11-21T11:14:35,070 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.4220 sec 2024-11-21T11:14:35,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:35,249 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 960784f3d73443a7fc815210ff1818b4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-21T11:14:35,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=A 2024-11-21T11:14:35,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:35,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=B 2024-11-21T11:14:35,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:35,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=C 2024-11-21T11:14:35,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:35,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112192f327aeb7134c4e95856c49b4a23f67_960784f3d73443a7fc815210ff1818b4 is 50, key is test_row_0/A:col10/1732187674571/Put/seqid=0 2024-11-21T11:14:35,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742062_1238 (size=14994) 2024-11-21T11:14:35,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:35,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187735276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:35,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:35,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:35,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:35,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:35,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187735277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:35,283 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:35,283 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:35,283 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:35,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:35,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187735282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:35,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:35,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:35,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:35,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:35,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187735283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:35,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:35,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:35,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:35,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:35,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187735385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:35,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:35,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:35,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:35,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187735385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:35,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:35,388 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:35,388 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:35,388 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:35,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:35,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187735396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:35,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:35,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:35,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187735396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:35,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:35,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:35,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:35,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:35,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:35,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:35,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187735595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:35,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:35,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187735595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:35,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:35,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:35,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:35,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:35,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:35,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:35,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:35,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:35,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187735606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:35,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187735606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:35,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:35,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:35,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:35,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:35,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:35,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:35,663 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:35,673 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112192f327aeb7134c4e95856c49b4a23f67_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112192f327aeb7134c4e95856c49b4a23f67_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:35,675 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/ef0b308537404f5487946516fbbdd2f4, store: [table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:35,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/ef0b308537404f5487946516fbbdd2f4 is 175, key is test_row_0/A:col10/1732187674571/Put/seqid=0 2024-11-21T11:14:35,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742063_1239 (size=39949) 2024-11-21T11:14:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T11:14:35,795 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:14:35,796 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:14:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-21T11:14:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-21T11:14:35,798 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:14:35,798 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:14:35,798 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:14:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-21T11:14:35,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:35,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187735906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:35,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:35,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:35,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:35,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187735917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:35,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187735917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:35,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:35,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:35,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:35,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187735918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:35,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:35,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:35,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:35,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:35,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:35,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:35,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-21T11:14:35,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:35,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:35,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:35,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:35,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:35,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:36,106 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=437, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/ef0b308537404f5487946516fbbdd2f4 2024-11-21T11:14:36,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-21T11:14:36,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:36,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:36,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:36,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:36,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:36,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-21T11:14:36,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/4d90cfac896b415599f3609f7ed9c7c5 is 50, key is test_row_0/B:col10/1732187674571/Put/seqid=0 2024-11-21T11:14:36,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742064_1240 (size=12301) 2024-11-21T11:14:36,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-21T11:14:36,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:36,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:36,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:36,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:36,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:36,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:36,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187736416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:36,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:36,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:36,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:36,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-21T11:14:36,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:36,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:36,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:36,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:36,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:36,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:36,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-21T11:14:36,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:36,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187736438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:36,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:36,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:36,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:36,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:36,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187736438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:36,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:36,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:36,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:36,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187736438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:36,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:36,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:36,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:36,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/4d90cfac896b415599f3609f7ed9c7c5 2024-11-21T11:14:36,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/c579e3234b8c4d2b84fa074ce8102469 is 50, key is test_row_0/C:col10/1732187674571/Put/seqid=0 2024-11-21T11:14:36,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-21T11:14:36,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:36,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. as already flushing 2024-11-21T11:14:36,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:36,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:36,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:36,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:36,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742065_1241 (size=12301) 2024-11-21T11:14:36,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/c579e3234b8c4d2b84fa074ce8102469 2024-11-21T11:14:36,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/ef0b308537404f5487946516fbbdd2f4 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/ef0b308537404f5487946516fbbdd2f4 2024-11-21T11:14:36,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/ef0b308537404f5487946516fbbdd2f4, entries=200, sequenceid=437, filesize=39.0 K 2024-11-21T11:14:36,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/4d90cfac896b415599f3609f7ed9c7c5 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/4d90cfac896b415599f3609f7ed9c7c5 2024-11-21T11:14:36,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/4d90cfac896b415599f3609f7ed9c7c5, entries=150, sequenceid=437, filesize=12.0 K 2024-11-21T11:14:36,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/c579e3234b8c4d2b84fa074ce8102469 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/c579e3234b8c4d2b84fa074ce8102469 2024-11-21T11:14:36,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/c579e3234b8c4d2b84fa074ce8102469, entries=150, sequenceid=437, filesize=12.0 K 2024-11-21T11:14:36,652 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 960784f3d73443a7fc815210ff1818b4 in 1403ms, sequenceid=437, compaction requested=true 2024-11-21T11:14:36,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:36,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 960784f3d73443a7fc815210ff1818b4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:14:36,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:36,653 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:14:36,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 960784f3d73443a7fc815210ff1818b4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:14:36,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:14:36,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 960784f3d73443a7fc815210ff1818b4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:14:36,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:14:36,653 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:14:36,655 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:14:36,655 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 960784f3d73443a7fc815210ff1818b4/B is initiating minor compaction (all files) 2024-11-21T11:14:36,655 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 960784f3d73443a7fc815210ff1818b4/B in TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:36,656 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/93e0f09f2cd3444eb1b5619d31045234, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/dc8cce0574174e6481f2120def6abd1f, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/4d90cfac896b415599f3609f7ed9c7c5] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp, totalSize=36.9 K 2024-11-21T11:14:36,656 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T11:14:36,657 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103345 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:14:36,657 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 960784f3d73443a7fc815210ff1818b4/A is initiating minor compaction (all files) 2024-11-21T11:14:36,657 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 960784f3d73443a7fc815210ff1818b4/A in TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:36,657 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/270b4a5d0c7046ba9b7d21d7e73a26ef, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/17faa47ac9da49148da712df321d3aa2, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/ef0b308537404f5487946516fbbdd2f4] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp, totalSize=100.9 K 2024-11-21T11:14:36,657 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:36,657 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. files: [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/270b4a5d0c7046ba9b7d21d7e73a26ef, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/17faa47ac9da49148da712df321d3aa2, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/ef0b308537404f5487946516fbbdd2f4] 2024-11-21T11:14:36,658 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 93e0f09f2cd3444eb1b5619d31045234, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732187673745 2024-11-21T11:14:36,658 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 270b4a5d0c7046ba9b7d21d7e73a26ef, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732187673745 2024-11-21T11:14:36,659 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17faa47ac9da49148da712df321d3aa2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1732187673885 2024-11-21T11:14:36,659 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting dc8cce0574174e6481f2120def6abd1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1732187673885 2024-11-21T11:14:36,661 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef0b308537404f5487946516fbbdd2f4, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1732187674547 2024-11-21T11:14:36,661 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d90cfac896b415599f3609f7ed9c7c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1732187674571 2024-11-21T11:14:36,693 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:36,696 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 960784f3d73443a7fc815210ff1818b4#B#compaction#198 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:36,697 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/c7de3fefaf2b477cb3016b1b4deaacb3 is 50, key is test_row_0/B:col10/1732187674571/Put/seqid=0 2024-11-21T11:14:36,712 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121fb2df5baebc94b469ec454c4299e5198_960784f3d73443a7fc815210ff1818b4 store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:36,715 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121fb2df5baebc94b469ec454c4299e5198_960784f3d73443a7fc815210ff1818b4, store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:36,715 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121fb2df5baebc94b469ec454c4299e5198_960784f3d73443a7fc815210ff1818b4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:36,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-21T11:14:36,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:36,732 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 960784f3d73443a7fc815210ff1818b4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-21T11:14:36,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=A 2024-11-21T11:14:36,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:36,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=B 2024-11-21T11:14:36,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:36,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=C 2024-11-21T11:14:36,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:36,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742066_1242 (size=13289) 2024-11-21T11:14:36,779 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/c7de3fefaf2b477cb3016b1b4deaacb3 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/c7de3fefaf2b477cb3016b1b4deaacb3 2024-11-21T11:14:36,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742067_1243 (size=4469) 2024-11-21T11:14:36,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219e25d317eaba4744a6e8456516a3c0fa_960784f3d73443a7fc815210ff1818b4 is 50, key is test_row_0/A:col10/1732187675281/Put/seqid=0 2024-11-21T11:14:36,801 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 960784f3d73443a7fc815210ff1818b4/B of 960784f3d73443a7fc815210ff1818b4 into c7de3fefaf2b477cb3016b1b4deaacb3(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:36,801 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:36,801 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., storeName=960784f3d73443a7fc815210ff1818b4/B, priority=13, startTime=1732187676653; duration=0sec 2024-11-21T11:14:36,801 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:14:36,801 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 960784f3d73443a7fc815210ff1818b4:B 2024-11-21T11:14:36,801 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:14:36,803 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:14:36,803 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 960784f3d73443a7fc815210ff1818b4/C is initiating minor compaction (all files) 2024-11-21T11:14:36,803 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 960784f3d73443a7fc815210ff1818b4/C in TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:36,803 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/a8d4e223c30646de8954f6b6f871e78c, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/ab295362759646a186cd85b515dd2123, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/c579e3234b8c4d2b84fa074ce8102469] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp, totalSize=36.9 K 2024-11-21T11:14:36,805 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting a8d4e223c30646de8954f6b6f871e78c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732187673745 2024-11-21T11:14:36,805 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting ab295362759646a186cd85b515dd2123, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1732187673885 2024-11-21T11:14:36,806 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting c579e3234b8c4d2b84fa074ce8102469, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1732187674571 2024-11-21T11:14:36,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742068_1244 (size=12454) 2024-11-21T11:14:36,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:36,831 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219e25d317eaba4744a6e8456516a3c0fa_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219e25d317eaba4744a6e8456516a3c0fa_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:36,833 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 960784f3d73443a7fc815210ff1818b4#C#compaction#200 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:36,834 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/f82089aa5c664624bb699cf136f133cb is 50, key is test_row_0/C:col10/1732187674571/Put/seqid=0 2024-11-21T11:14:36,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/565a0e46a3fc4221bd49f4d418958510, store: [table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:36,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/565a0e46a3fc4221bd49f4d418958510 is 175, key is test_row_0/A:col10/1732187675281/Put/seqid=0 2024-11-21T11:14:36,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742069_1245 (size=13255) 2024-11-21T11:14:36,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742070_1246 (size=31255) 2024-11-21T11:14:36,860 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=457, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/565a0e46a3fc4221bd49f4d418958510 2024-11-21T11:14:36,870 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/f82089aa5c664624bb699cf136f133cb as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/f82089aa5c664624bb699cf136f133cb 2024-11-21T11:14:36,877 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 960784f3d73443a7fc815210ff1818b4/C of 960784f3d73443a7fc815210ff1818b4 into f82089aa5c664624bb699cf136f133cb(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:36,877 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:36,877 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., storeName=960784f3d73443a7fc815210ff1818b4/C, priority=13, startTime=1732187676653; duration=0sec 2024-11-21T11:14:36,877 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:36,877 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 960784f3d73443a7fc815210ff1818b4:C 2024-11-21T11:14:36,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/f062497be3f94498b3b45d5a4eed80d6 is 50, key is test_row_0/B:col10/1732187675281/Put/seqid=0 2024-11-21T11:14:36,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742071_1247 (size=12301) 2024-11-21T11:14:36,910 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/f062497be3f94498b3b45d5a4eed80d6 2024-11-21T11:14:36,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/b6556184e5c04777b741176369fd1b10 is 50, key is test_row_0/C:col10/1732187675281/Put/seqid=0 2024-11-21T11:14:36,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742072_1248 (size=12301) 2024-11-21T11:14:36,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-21T11:14:37,190 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 960784f3d73443a7fc815210ff1818b4#A#compaction#197 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:37,191 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/7de94f79c608450f8ab3d0141cbec4ba is 175, key is test_row_0/A:col10/1732187674571/Put/seqid=0 2024-11-21T11:14:37,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742073_1249 (size=32243) 2024-11-21T11:14:37,210 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/7de94f79c608450f8ab3d0141cbec4ba as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/7de94f79c608450f8ab3d0141cbec4ba 2024-11-21T11:14:37,219 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 960784f3d73443a7fc815210ff1818b4/A of 960784f3d73443a7fc815210ff1818b4 into 7de94f79c608450f8ab3d0141cbec4ba(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:37,220 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:37,220 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., storeName=960784f3d73443a7fc815210ff1818b4/A, priority=13, startTime=1732187676653; duration=0sec 2024-11-21T11:14:37,220 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:37,220 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 960784f3d73443a7fc815210ff1818b4:A 2024-11-21T11:14:37,334 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/b6556184e5c04777b741176369fd1b10 2024-11-21T11:14:37,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/565a0e46a3fc4221bd49f4d418958510 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/565a0e46a3fc4221bd49f4d418958510 2024-11-21T11:14:37,346 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/565a0e46a3fc4221bd49f4d418958510, entries=150, sequenceid=457, filesize=30.5 K 2024-11-21T11:14:37,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/f062497be3f94498b3b45d5a4eed80d6 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/f062497be3f94498b3b45d5a4eed80d6 2024-11-21T11:14:37,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,363 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/f062497be3f94498b3b45d5a4eed80d6, entries=150, sequenceid=457, filesize=12.0 K 2024-11-21T11:14:37,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/b6556184e5c04777b741176369fd1b10 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/b6556184e5c04777b741176369fd1b10 2024-11-21T11:14:37,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,370 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/b6556184e5c04777b741176369fd1b10, entries=150, sequenceid=457, filesize=12.0 K 2024-11-21T11:14:37,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,371 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 960784f3d73443a7fc815210ff1818b4 in 640ms, sequenceid=457, compaction requested=false 2024-11-21T11:14:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:37,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:37,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-21T11:14:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-21T11:14:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,375 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-21T11:14:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,375 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5750 sec 2024-11-21T11:14:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,377 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.5800 sec 2024-11-21T11:14:37,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 960784f3d73443a7fc815210ff1818b4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T11:14:37,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=A 2024-11-21T11:14:37,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:37,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=B 2024-11-21T11:14:37,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:37,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=C 2024-11-21T11:14:37,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:37,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:37,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121bd571334ef684175a692e19f4617885f_960784f3d73443a7fc815210ff1818b4 is 50, key is test_row_0/A:col10/1732187677482/Put/seqid=0 2024-11-21T11:14:37,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742074_1250 (size=22618) 2024-11-21T11:14:37,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,520 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,524 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121bd571334ef684175a692e19f4617885f_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121bd571334ef684175a692e19f4617885f_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:37,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,525 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/5cbc8bff40a742aeaff2b4c04a69c4f5, store: [table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:37,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/5cbc8bff40a742aeaff2b4c04a69c4f5 is 175, key is test_row_0/A:col10/1732187677482/Put/seqid=0 2024-11-21T11:14:37,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:37,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742075_1251 (size=66023) 2024-11-21T11:14:37,545 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=470, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/5cbc8bff40a742aeaff2b4c04a69c4f5 2024-11-21T11:14:37,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/aa880840d4eb44bdb85983effe8f86b3 is 50, key is test_row_0/B:col10/1732187677482/Put/seqid=0 2024-11-21T11:14:37,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732187677541, max timestamp=1732187677557Num uniques 859; 2024-11-21T11:14:37,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T11:14:37,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732187677541, max timestamp=1732187677557Num uniques 859; 2024-11-21T11:14:37,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T11:14:37,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732187677541, max timestamp=1732187677559Num uniques 859; 2024-11-21T11:14:37,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T11:14:37,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742076_1252 (size=12301) 2024-11-21T11:14:37,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:37,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187737583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:37,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:37,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:37,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187737585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:37,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:37,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:37,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:37,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:37,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:37,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:37,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732187737586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:37,589 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_0' in 960784f3d73443a7fc815210ff1818b4 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18314 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:14:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:37,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:37,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187737586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:37,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:37,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:37,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:37,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:37,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187737588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:37,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:37,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:37,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:37,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:37,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187737695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:37,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:37,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:37,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187737695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:37,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:37,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:37,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:37,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:37,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:37,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187737695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:37,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:37,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:37,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:37,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:37,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:37,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187737705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:37,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:37,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:37,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:37,832 INFO [Thread-646 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-646. 2024-11-21T11:14:37,832 DEBUG [Thread-646 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:14:37,832 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:37,833 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:37,833 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:14:37,836 INFO [Thread-652 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-652. 2024-11-21T11:14:37,836 DEBUG [Thread-652 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:14:37,836 DEBUG [Thread-652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:37,836 DEBUG [Thread-652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:37,836 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:14:37,837 INFO [Thread-648 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-648. 2024-11-21T11:14:37,837 DEBUG [Thread-648 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:14:37,837 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:37,838 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:37,838 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:14:37,838 INFO [Thread-650 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-650. 2024-11-21T11:14:37,838 DEBUG [Thread-650 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:14:37,838 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:37,838 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:37,838 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:14:37,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:37,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:37,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:37,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187737905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:37,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187737905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:37,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187737905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:37,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:37,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:37,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:37,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:37,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:37,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:37,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:37,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:37,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:37,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:37,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187737916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:37,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:37,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:37,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:37,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-21T11:14:37,945 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:14:37,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/aa880840d4eb44bdb85983effe8f86b3 2024-11-21T11:14:37,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/ab04bdaa50aa4f7ab1176e28b423ff39 is 50, key is test_row_0/C:col10/1732187677482/Put/seqid=0 2024-11-21T11:14:37,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742077_1253 (size=12301) 2024-11-21T11:14:38,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:38,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:38,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:38,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60558 deadline: 1732187738216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:38,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1732187738216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:38,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60504 deadline: 1732187738216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:38,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:38,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:38,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:38,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:38,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:38,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:38,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:38,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:38,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:38,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:38,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60548 deadline: 1732187738225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:38,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:38,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:38,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:14:38,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/ab04bdaa50aa4f7ab1176e28b423ff39 2024-11-21T11:14:38,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/5cbc8bff40a742aeaff2b4c04a69c4f5 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/5cbc8bff40a742aeaff2b4c04a69c4f5 2024-11-21T11:14:38,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/5cbc8bff40a742aeaff2b4c04a69c4f5, entries=350, sequenceid=470, filesize=64.5 K 2024-11-21T11:14:38,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/aa880840d4eb44bdb85983effe8f86b3 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/aa880840d4eb44bdb85983effe8f86b3 2024-11-21T11:14:38,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/aa880840d4eb44bdb85983effe8f86b3, entries=150, sequenceid=470, filesize=12.0 K 2024-11-21T11:14:38,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/ab04bdaa50aa4f7ab1176e28b423ff39 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/ab04bdaa50aa4f7ab1176e28b423ff39 2024-11-21T11:14:38,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/ab04bdaa50aa4f7ab1176e28b423ff39, entries=150, sequenceid=470, filesize=12.0 K 2024-11-21T11:14:38,396 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 960784f3d73443a7fc815210ff1818b4 in 905ms, sequenceid=470, compaction requested=true 2024-11-21T11:14:38,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:38,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 960784f3d73443a7fc815210ff1818b4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:14:38,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:38,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 960784f3d73443a7fc815210ff1818b4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:14:38,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:38,396 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:14:38,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 960784f3d73443a7fc815210ff1818b4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:14:38,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:14:38,396 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:14:38,397 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 129521 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:14:38,397 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:14:38,397 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 960784f3d73443a7fc815210ff1818b4/A is initiating minor compaction (all files) 2024-11-21T11:14:38,397 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 960784f3d73443a7fc815210ff1818b4/B is initiating minor compaction (all files) 2024-11-21T11:14:38,397 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 960784f3d73443a7fc815210ff1818b4/A in TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:38,397 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 960784f3d73443a7fc815210ff1818b4/B in TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:38,397 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/7de94f79c608450f8ab3d0141cbec4ba, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/565a0e46a3fc4221bd49f4d418958510, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/5cbc8bff40a742aeaff2b4c04a69c4f5] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp, totalSize=126.5 K 2024-11-21T11:14:38,397 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/c7de3fefaf2b477cb3016b1b4deaacb3, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/f062497be3f94498b3b45d5a4eed80d6, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/aa880840d4eb44bdb85983effe8f86b3] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp, totalSize=37.0 K 2024-11-21T11:14:38,397 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:38,397 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. files: [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/7de94f79c608450f8ab3d0141cbec4ba, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/565a0e46a3fc4221bd49f4d418958510, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/5cbc8bff40a742aeaff2b4c04a69c4f5] 2024-11-21T11:14:38,397 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting c7de3fefaf2b477cb3016b1b4deaacb3, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1732187674571 2024-11-21T11:14:38,397 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7de94f79c608450f8ab3d0141cbec4ba, keycount=150, bloomtype=ROW, size=31.5 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1732187674571 2024-11-21T11:14:38,398 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting f062497be3f94498b3b45d5a4eed80d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1732187675270 2024-11-21T11:14:38,398 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 565a0e46a3fc4221bd49f4d418958510, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1732187675270 2024-11-21T11:14:38,398 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting aa880840d4eb44bdb85983effe8f86b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1732187677480 2024-11-21T11:14:38,398 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cbc8bff40a742aeaff2b4c04a69c4f5, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1732187677473 2024-11-21T11:14:38,407 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 960784f3d73443a7fc815210ff1818b4#B#compaction#206 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:38,407 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:38,408 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/9608e3cea86a49be862c58071bca4c4e is 50, key is test_row_0/B:col10/1732187677482/Put/seqid=0 2024-11-21T11:14:38,409 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121266e5425d5964ba586c2986c1455916d_960784f3d73443a7fc815210ff1818b4 store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:38,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742078_1254 (size=13391) 2024-11-21T11:14:38,429 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121266e5425d5964ba586c2986c1455916d_960784f3d73443a7fc815210ff1818b4, store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:38,430 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121266e5425d5964ba586c2986c1455916d_960784f3d73443a7fc815210ff1818b4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:38,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742079_1255 (size=4469) 2024-11-21T11:14:38,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:38,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 960784f3d73443a7fc815210ff1818b4 3/3 column families, dataSize=154.31 KB heapSize=440.77 KB 2024-11-21T11:14:38,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=A 2024-11-21T11:14:38,729 INFO [Thread-643 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-643. 2024-11-21T11:14:38,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-21T11:14:38,729 DEBUG [Thread-643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:14:38,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=B 2024-11-21T11:14:38,729 DEBUG [Thread-643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:38,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-21T11:14:38,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=C 2024-11-21T11:14:38,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-21T11:14:38,729 DEBUG [Thread-643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:38,729 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:14:38,729 INFO [Thread-641 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-641. 2024-11-21T11:14:38,729 DEBUG [Thread-641 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:14:38,729 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:38,729 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:38,731 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:14:38,735 INFO [Thread-637 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-637. 2024-11-21T11:14:38,735 DEBUG [Thread-637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:14:38,735 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:38,735 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:38,735 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:14:38,738 INFO [Thread-635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-635. 2024-11-21T11:14:38,738 DEBUG [Thread-635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:14:38,738 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:38,738 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:38,738 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:14:38,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214f11fe6d28e74f62b364b28416de2bdf_960784f3d73443a7fc815210ff1818b4 is 50, key is test_row_0/A:col10/1732187678725/Put/seqid=0 2024-11-21T11:14:38,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742080_1256 (size=12454) 2024-11-21T11:14:38,772 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:38,777 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214f11fe6d28e74f62b364b28416de2bdf_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214f11fe6d28e74f62b364b28416de2bdf_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:38,782 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/f281bdcf6bd748dd9523bde9f3b776cd, store: [table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:38,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/f281bdcf6bd748dd9523bde9f3b776cd is 175, key is test_row_0/A:col10/1732187678725/Put/seqid=0 2024-11-21T11:14:38,819 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/9608e3cea86a49be862c58071bca4c4e as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/9608e3cea86a49be862c58071bca4c4e 2024-11-21T11:14:38,830 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 960784f3d73443a7fc815210ff1818b4/B of 960784f3d73443a7fc815210ff1818b4 into 9608e3cea86a49be862c58071bca4c4e(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:38,830 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:38,830 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., storeName=960784f3d73443a7fc815210ff1818b4/B, priority=13, startTime=1732187678396; duration=0sec 2024-11-21T11:14:38,830 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:14:38,830 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 960784f3d73443a7fc815210ff1818b4:B 2024-11-21T11:14:38,830 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:14:38,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742081_1257 (size=31255) 2024-11-21T11:14:38,832 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:14:38,832 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 960784f3d73443a7fc815210ff1818b4/C is initiating minor compaction (all files) 2024-11-21T11:14:38,832 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 960784f3d73443a7fc815210ff1818b4/C in TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:38,832 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/f82089aa5c664624bb699cf136f133cb, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/b6556184e5c04777b741176369fd1b10, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/ab04bdaa50aa4f7ab1176e28b423ff39] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp, totalSize=37.0 K 2024-11-21T11:14:38,833 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting f82089aa5c664624bb699cf136f133cb, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1732187674571 2024-11-21T11:14:38,833 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=509, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/f281bdcf6bd748dd9523bde9f3b776cd 2024-11-21T11:14:38,833 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting b6556184e5c04777b741176369fd1b10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1732187675270 2024-11-21T11:14:38,834 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting ab04bdaa50aa4f7ab1176e28b423ff39, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1732187677480 2024-11-21T11:14:38,837 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 960784f3d73443a7fc815210ff1818b4#A#compaction#207 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:38,839 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/69864eb816b34092a86056f7dac92923 is 175, key is test_row_0/A:col10/1732187677482/Put/seqid=0 2024-11-21T11:14:38,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/13ac38f3cd7a4bd884d59faddda48105 is 50, key is test_row_0/B:col10/1732187678725/Put/seqid=0 2024-11-21T11:14:38,862 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 960784f3d73443a7fc815210ff1818b4#C#compaction#210 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:38,863 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/31041caab1cf497298515371f0716766 is 50, key is test_row_0/C:col10/1732187677482/Put/seqid=0 2024-11-21T11:14:38,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742082_1258 (size=32345) 2024-11-21T11:14:38,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742084_1260 (size=13357) 2024-11-21T11:14:38,906 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/31041caab1cf497298515371f0716766 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/31041caab1cf497298515371f0716766 2024-11-21T11:14:38,913 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 960784f3d73443a7fc815210ff1818b4/C of 960784f3d73443a7fc815210ff1818b4 into 31041caab1cf497298515371f0716766(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:38,913 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:38,913 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., storeName=960784f3d73443a7fc815210ff1818b4/C, priority=13, startTime=1732187678396; duration=0sec 2024-11-21T11:14:38,913 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:38,913 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 960784f3d73443a7fc815210ff1818b4:C 2024-11-21T11:14:38,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742083_1259 (size=12301) 2024-11-21T11:14:38,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/13ac38f3cd7a4bd884d59faddda48105 2024-11-21T11:14:38,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/4f01ef00b5094af8ac444d1f7fb1a691 is 50, key is test_row_0/C:col10/1732187678725/Put/seqid=0 2024-11-21T11:14:38,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742085_1261 (size=12301) 2024-11-21T11:14:39,280 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/69864eb816b34092a86056f7dac92923 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/69864eb816b34092a86056f7dac92923 2024-11-21T11:14:39,285 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 960784f3d73443a7fc815210ff1818b4/A of 960784f3d73443a7fc815210ff1818b4 into 69864eb816b34092a86056f7dac92923(size=31.6 K), total size for store is 31.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:39,285 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:39,285 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., storeName=960784f3d73443a7fc815210ff1818b4/A, priority=13, startTime=1732187678396; duration=0sec 2024-11-21T11:14:39,285 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:39,285 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 960784f3d73443a7fc815210ff1818b4:A 2024-11-21T11:14:39,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/4f01ef00b5094af8ac444d1f7fb1a691 2024-11-21T11:14:39,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/f281bdcf6bd748dd9523bde9f3b776cd as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/f281bdcf6bd748dd9523bde9f3b776cd 2024-11-21T11:14:39,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/f281bdcf6bd748dd9523bde9f3b776cd, entries=150, sequenceid=509, filesize=30.5 K 2024-11-21T11:14:39,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/13ac38f3cd7a4bd884d59faddda48105 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/13ac38f3cd7a4bd884d59faddda48105 2024-11-21T11:14:39,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/13ac38f3cd7a4bd884d59faddda48105, entries=150, sequenceid=509, filesize=12.0 K 2024-11-21T11:14:39,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/4f01ef00b5094af8ac444d1f7fb1a691 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/4f01ef00b5094af8ac444d1f7fb1a691 2024-11-21T11:14:39,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/4f01ef00b5094af8ac444d1f7fb1a691, entries=150, sequenceid=509, filesize=12.0 K 2024-11-21T11:14:39,371 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~440.72 KB/451296, currentSize=6.71 KB/6870 for 960784f3d73443a7fc815210ff1818b4 in 643ms, sequenceid=509, compaction requested=false 2024-11-21T11:14:39,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 960784f3d73443a7fc815210ff1818b4: 2024-11-21T11:14:47,607 INFO [Thread-639 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-639. 2024-11-21T11:14:47,607 DEBUG [Thread-639 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:14:47,607 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:47,607 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:47,607 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:14:47,607 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T11:14:47,607 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 114 2024-11-21T11:14:47,607 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 101 2024-11-21T11:14:47,607 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-21T11:14:47,607 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-21T11:14:47,607 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 110 2024-11-21T11:14:47,607 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T11:14:47,607 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2625 2024-11-21T11:14:47,607 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2541 2024-11-21T11:14:47,607 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T11:14:47,607 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1122 2024-11-21T11:14:47,607 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3366 rows 2024-11-21T11:14:47,607 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1132 2024-11-21T11:14:47,607 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3394 rows 2024-11-21T11:14:47,608 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T11:14:47,608 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T11:14:47,608 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:47,608 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:47,608 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:14:47,609 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T11:14:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T11:14:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-21T11:14:47,612 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732187687612"}]},"ts":"1732187687612"} 2024-11-21T11:14:47,614 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T11:14:47,614 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T11:14:47,615 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T11:14:47,616 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=960784f3d73443a7fc815210ff1818b4, UNASSIGN}] 2024-11-21T11:14:47,617 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=960784f3d73443a7fc815210ff1818b4, UNASSIGN 2024-11-21T11:14:47,618 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=960784f3d73443a7fc815210ff1818b4, regionState=CLOSING, regionLocation=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:47,620 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=960784f3d73443a7fc815210ff1818b4, UNASSIGN because future has completed 2024-11-21T11:14:47,620 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T11:14:47,620 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure 960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276}] 2024-11-21T11:14:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-21T11:14:47,773 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(122): Close 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:47,774 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T11:14:47,774 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1722): Closing 960784f3d73443a7fc815210ff1818b4, disabling compactions & flushes 2024-11-21T11:14:47,774 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:47,774 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:47,774 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. after waiting 0 ms 2024-11-21T11:14:47,774 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:47,774 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(2902): Flushing 960784f3d73443a7fc815210ff1818b4 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-21T11:14:47,774 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=A 2024-11-21T11:14:47,774 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:47,774 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=B 2024-11-21T11:14:47,774 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:47,774 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 960784f3d73443a7fc815210ff1818b4, store=C 2024-11-21T11:14:47,774 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:47,786 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b643a4a94ee0423cb05c8633c2e132b1_960784f3d73443a7fc815210ff1818b4 is 50, key is test_row_0/A:col10/1732187687605/Put/seqid=0 2024-11-21T11:14:47,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742086_1262 (size=9914) 2024-11-21T11:14:47,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-21T11:14:48,193 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:48,197 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b643a4a94ee0423cb05c8633c2e132b1_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b643a4a94ee0423cb05c8633c2e132b1_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:48,198 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/57ee4acccc0d4b50925f2528a3ae299f, store: [table=TestAcidGuarantees family=A region=960784f3d73443a7fc815210ff1818b4] 2024-11-21T11:14:48,198 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/57ee4acccc0d4b50925f2528a3ae299f is 175, key is test_row_0/A:col10/1732187687605/Put/seqid=0 2024-11-21T11:14:48,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742087_1263 (size=22561) 2024-11-21T11:14:48,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-21T11:14:48,602 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=517, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/57ee4acccc0d4b50925f2528a3ae299f 2024-11-21T11:14:48,609 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/3e8293810a8b4dae93a6873e42184436 is 50, key is test_row_0/B:col10/1732187687605/Put/seqid=0 2024-11-21T11:14:48,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742088_1264 (size=9857) 2024-11-21T11:14:48,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-21T11:14:49,013 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/3e8293810a8b4dae93a6873e42184436 2024-11-21T11:14:49,021 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/7b2594b2623a477ca0ae2a1fd16d95de is 50, key is test_row_0/C:col10/1732187687605/Put/seqid=0 2024-11-21T11:14:49,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742089_1265 (size=9857) 2024-11-21T11:14:49,425 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/7b2594b2623a477ca0ae2a1fd16d95de 2024-11-21T11:14:49,432 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/A/57ee4acccc0d4b50925f2528a3ae299f as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/57ee4acccc0d4b50925f2528a3ae299f 2024-11-21T11:14:49,440 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/57ee4acccc0d4b50925f2528a3ae299f, entries=100, sequenceid=517, filesize=22.0 K 2024-11-21T11:14:49,441 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/B/3e8293810a8b4dae93a6873e42184436 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/3e8293810a8b4dae93a6873e42184436 2024-11-21T11:14:49,446 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/3e8293810a8b4dae93a6873e42184436, entries=100, sequenceid=517, filesize=9.6 K 2024-11-21T11:14:49,447 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/.tmp/C/7b2594b2623a477ca0ae2a1fd16d95de as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/7b2594b2623a477ca0ae2a1fd16d95de 2024-11-21T11:14:49,451 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/7b2594b2623a477ca0ae2a1fd16d95de, entries=100, sequenceid=517, filesize=9.6 K 2024-11-21T11:14:49,452 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 960784f3d73443a7fc815210ff1818b4 in 1678ms, sequenceid=517, compaction requested=true 2024-11-21T11:14:49,453 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/cba0e1b407754cf79812527522409715, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/f9e198556ddb4cc688a6ea95880bae58, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/4cfd88eb5ba849b684c9c5eb3b027f6b, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/6f5a18fcfb6a4174bf8d53ee665b309a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/7f81958bf0d84d178cd4dbc9831b9b15, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/17e9f486798a411e875de69412baadf3, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/26aa176b776b4f739f05a40e8d2dd61a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/07ce0aa3fd6e401fa9184db6ec8f34ff, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/fed607b2986245c79ec041225ae14e5a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/94c591f009de46d09b2083006ecc478d, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/91c0c5e9a44a4f4eb1bc67b470185999, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/e5e0a73b00d546cbac6f165fc226f8a9, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/1d80620e283c4e0d89fe0304a03b52d4, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/e822c0660da149a7a69040e77aa51296, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/29b8abe74cb44a61b4939dd5a63101e8, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/e8d36b01d135480897fc379a13e8b6c3, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/23fa475952e741858588ec20a4c80956, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/92c46b58e47f43619f10a7474a046d1d, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/defa2e0ea44f4f89b5f0d26622a36d5c, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/4a45788f1e334f6ebb83ec8a1c59a18e, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/e3595e8d0c29427086a82259eb52e26a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/d28f73bf45044b89a6174f007c701cfb, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/9d6617d39ede47d3b17b39d3e336b12b, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/0400606c8d544027a10e8dc71c7e8fa5, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/1412abb3e13e447b8d01c80c9cf2a887, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/9bbc0b2575ef495ba77667e2442be918, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/270b4a5d0c7046ba9b7d21d7e73a26ef, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/17faa47ac9da49148da712df321d3aa2, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/ef0b308537404f5487946516fbbdd2f4, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/7de94f79c608450f8ab3d0141cbec4ba, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/565a0e46a3fc4221bd49f4d418958510, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/5cbc8bff40a742aeaff2b4c04a69c4f5] to archive 2024-11-21T11:14:49,454 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T11:14:49,456 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/cba0e1b407754cf79812527522409715 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/cba0e1b407754cf79812527522409715 2024-11-21T11:14:49,457 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/f9e198556ddb4cc688a6ea95880bae58 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/f9e198556ddb4cc688a6ea95880bae58 2024-11-21T11:14:49,459 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/4cfd88eb5ba849b684c9c5eb3b027f6b to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/4cfd88eb5ba849b684c9c5eb3b027f6b 2024-11-21T11:14:49,460 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/6f5a18fcfb6a4174bf8d53ee665b309a to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/6f5a18fcfb6a4174bf8d53ee665b309a 2024-11-21T11:14:49,462 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/7f81958bf0d84d178cd4dbc9831b9b15 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/7f81958bf0d84d178cd4dbc9831b9b15 2024-11-21T11:14:49,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/17e9f486798a411e875de69412baadf3 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/17e9f486798a411e875de69412baadf3 2024-11-21T11:14:49,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/26aa176b776b4f739f05a40e8d2dd61a to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/26aa176b776b4f739f05a40e8d2dd61a 2024-11-21T11:14:49,469 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/07ce0aa3fd6e401fa9184db6ec8f34ff to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/07ce0aa3fd6e401fa9184db6ec8f34ff 2024-11-21T11:14:49,471 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/fed607b2986245c79ec041225ae14e5a to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/fed607b2986245c79ec041225ae14e5a 2024-11-21T11:14:49,473 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/94c591f009de46d09b2083006ecc478d to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/94c591f009de46d09b2083006ecc478d 2024-11-21T11:14:49,474 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/91c0c5e9a44a4f4eb1bc67b470185999 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/91c0c5e9a44a4f4eb1bc67b470185999 2024-11-21T11:14:49,475 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/e5e0a73b00d546cbac6f165fc226f8a9 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/e5e0a73b00d546cbac6f165fc226f8a9 2024-11-21T11:14:49,476 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/1d80620e283c4e0d89fe0304a03b52d4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/1d80620e283c4e0d89fe0304a03b52d4 2024-11-21T11:14:49,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/e822c0660da149a7a69040e77aa51296 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/e822c0660da149a7a69040e77aa51296 2024-11-21T11:14:49,479 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/29b8abe74cb44a61b4939dd5a63101e8 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/29b8abe74cb44a61b4939dd5a63101e8 2024-11-21T11:14:49,480 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/e8d36b01d135480897fc379a13e8b6c3 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/e8d36b01d135480897fc379a13e8b6c3 2024-11-21T11:14:49,482 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/23fa475952e741858588ec20a4c80956 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/23fa475952e741858588ec20a4c80956 2024-11-21T11:14:49,483 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/92c46b58e47f43619f10a7474a046d1d to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/92c46b58e47f43619f10a7474a046d1d 2024-11-21T11:14:49,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/defa2e0ea44f4f89b5f0d26622a36d5c to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/defa2e0ea44f4f89b5f0d26622a36d5c 2024-11-21T11:14:49,485 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/4a45788f1e334f6ebb83ec8a1c59a18e to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/4a45788f1e334f6ebb83ec8a1c59a18e 2024-11-21T11:14:49,487 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/e3595e8d0c29427086a82259eb52e26a to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/e3595e8d0c29427086a82259eb52e26a 2024-11-21T11:14:49,488 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/d28f73bf45044b89a6174f007c701cfb to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/d28f73bf45044b89a6174f007c701cfb 2024-11-21T11:14:49,489 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/9d6617d39ede47d3b17b39d3e336b12b to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/9d6617d39ede47d3b17b39d3e336b12b 2024-11-21T11:14:49,490 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/0400606c8d544027a10e8dc71c7e8fa5 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/0400606c8d544027a10e8dc71c7e8fa5 2024-11-21T11:14:49,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/1412abb3e13e447b8d01c80c9cf2a887 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/1412abb3e13e447b8d01c80c9cf2a887 2024-11-21T11:14:49,493 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/9bbc0b2575ef495ba77667e2442be918 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/9bbc0b2575ef495ba77667e2442be918 2024-11-21T11:14:49,494 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/270b4a5d0c7046ba9b7d21d7e73a26ef to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/270b4a5d0c7046ba9b7d21d7e73a26ef 2024-11-21T11:14:49,495 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/17faa47ac9da49148da712df321d3aa2 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/17faa47ac9da49148da712df321d3aa2 2024-11-21T11:14:49,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/ef0b308537404f5487946516fbbdd2f4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/ef0b308537404f5487946516fbbdd2f4 2024-11-21T11:14:49,497 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/7de94f79c608450f8ab3d0141cbec4ba to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/7de94f79c608450f8ab3d0141cbec4ba 2024-11-21T11:14:49,498 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/565a0e46a3fc4221bd49f4d418958510 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/565a0e46a3fc4221bd49f4d418958510 2024-11-21T11:14:49,499 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/5cbc8bff40a742aeaff2b4c04a69c4f5 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/5cbc8bff40a742aeaff2b4c04a69c4f5 2024-11-21T11:14:49,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/094f3cb0ae3e4c5994c15e6a1dee530b, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/927beefde1e64b81987b3f5fbf8757ab, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/0dbac1aba0f94ca087ae636116231d9a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/499658ddf97143f7bfce661b2cf81e3e, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/5fdf229b277c461fa30e78e48ee61220, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/bdb56f0412eb4952863af668a88fd324, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/174b528c8da044a09a92de9b56dd3b00, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/a7419d2819ef462db99eeb9194d75b13, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/759319bb5f7b4ce7a4957850b31798b3, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/e103786a84c543e3912ba1a9093a7a1d, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/0a7b91bf6038477cbd1c9b0502b47144, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/c34858f38d554857a9eeea1b5cc22e8a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/aa3c24978c0547a88c36825bb4691b1d, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/89460b4dac574c7aab9e31041e645023, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/94c15e07870d41d885d7e3892f1bf62c, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/81022537f5984a0d96bf5d9a93b53c1c, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/64c0c83f0f2c4551a9f0369e289ba7a4, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/4a69a46211c7434c95165dff6028b02c, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/301b79fe32c34f8d9b4d9adfac0c6807, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/5ec4ea9d88934347a31074279ae03673, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/4ea154aa87a4431295edcff73bdc3ffa, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/cd9ff6588ca244a2bc6616f9b5cd89aa, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/29153c0d2f5f430483f57fb5deef3ecd, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/912ecd0ab20f44e2a412847e8ff6d069, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/7a6525114bb34edebfdd4006a3d64840, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/93e0f09f2cd3444eb1b5619d31045234, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/211421efd3cf482d935afd59bcc69089, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/dc8cce0574174e6481f2120def6abd1f, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/c7de3fefaf2b477cb3016b1b4deaacb3, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/4d90cfac896b415599f3609f7ed9c7c5, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/f062497be3f94498b3b45d5a4eed80d6, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/aa880840d4eb44bdb85983effe8f86b3] to archive 2024-11-21T11:14:49,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T11:14:49,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/094f3cb0ae3e4c5994c15e6a1dee530b to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/094f3cb0ae3e4c5994c15e6a1dee530b 2024-11-21T11:14:49,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/927beefde1e64b81987b3f5fbf8757ab to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/927beefde1e64b81987b3f5fbf8757ab 2024-11-21T11:14:49,506 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/0dbac1aba0f94ca087ae636116231d9a to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/0dbac1aba0f94ca087ae636116231d9a 2024-11-21T11:14:49,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/499658ddf97143f7bfce661b2cf81e3e to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/499658ddf97143f7bfce661b2cf81e3e 2024-11-21T11:14:49,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/5fdf229b277c461fa30e78e48ee61220 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/5fdf229b277c461fa30e78e48ee61220 2024-11-21T11:14:49,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/bdb56f0412eb4952863af668a88fd324 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/bdb56f0412eb4952863af668a88fd324 2024-11-21T11:14:49,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/174b528c8da044a09a92de9b56dd3b00 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/174b528c8da044a09a92de9b56dd3b00 2024-11-21T11:14:49,512 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/a7419d2819ef462db99eeb9194d75b13 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/a7419d2819ef462db99eeb9194d75b13 2024-11-21T11:14:49,513 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/759319bb5f7b4ce7a4957850b31798b3 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/759319bb5f7b4ce7a4957850b31798b3 2024-11-21T11:14:49,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/e103786a84c543e3912ba1a9093a7a1d to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/e103786a84c543e3912ba1a9093a7a1d 2024-11-21T11:14:49,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/0a7b91bf6038477cbd1c9b0502b47144 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/0a7b91bf6038477cbd1c9b0502b47144 2024-11-21T11:14:49,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/c34858f38d554857a9eeea1b5cc22e8a to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/c34858f38d554857a9eeea1b5cc22e8a 2024-11-21T11:14:49,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/aa3c24978c0547a88c36825bb4691b1d to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/aa3c24978c0547a88c36825bb4691b1d 2024-11-21T11:14:49,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/89460b4dac574c7aab9e31041e645023 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/89460b4dac574c7aab9e31041e645023 2024-11-21T11:14:49,522 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/94c15e07870d41d885d7e3892f1bf62c to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/94c15e07870d41d885d7e3892f1bf62c 2024-11-21T11:14:49,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/81022537f5984a0d96bf5d9a93b53c1c to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/81022537f5984a0d96bf5d9a93b53c1c 2024-11-21T11:14:49,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/64c0c83f0f2c4551a9f0369e289ba7a4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/64c0c83f0f2c4551a9f0369e289ba7a4 2024-11-21T11:14:49,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/4a69a46211c7434c95165dff6028b02c to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/4a69a46211c7434c95165dff6028b02c 2024-11-21T11:14:49,527 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/301b79fe32c34f8d9b4d9adfac0c6807 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/301b79fe32c34f8d9b4d9adfac0c6807 2024-11-21T11:14:49,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/5ec4ea9d88934347a31074279ae03673 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/5ec4ea9d88934347a31074279ae03673 2024-11-21T11:14:49,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/4ea154aa87a4431295edcff73bdc3ffa to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/4ea154aa87a4431295edcff73bdc3ffa 2024-11-21T11:14:49,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/cd9ff6588ca244a2bc6616f9b5cd89aa to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/cd9ff6588ca244a2bc6616f9b5cd89aa 2024-11-21T11:14:49,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/29153c0d2f5f430483f57fb5deef3ecd to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/29153c0d2f5f430483f57fb5deef3ecd 2024-11-21T11:14:49,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/912ecd0ab20f44e2a412847e8ff6d069 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/912ecd0ab20f44e2a412847e8ff6d069 2024-11-21T11:14:49,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/7a6525114bb34edebfdd4006a3d64840 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/7a6525114bb34edebfdd4006a3d64840 2024-11-21T11:14:49,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/93e0f09f2cd3444eb1b5619d31045234 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/93e0f09f2cd3444eb1b5619d31045234 2024-11-21T11:14:49,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/211421efd3cf482d935afd59bcc69089 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/211421efd3cf482d935afd59bcc69089 2024-11-21T11:14:49,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/dc8cce0574174e6481f2120def6abd1f to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/dc8cce0574174e6481f2120def6abd1f 2024-11-21T11:14:49,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/c7de3fefaf2b477cb3016b1b4deaacb3 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/c7de3fefaf2b477cb3016b1b4deaacb3 2024-11-21T11:14:49,542 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/4d90cfac896b415599f3609f7ed9c7c5 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/4d90cfac896b415599f3609f7ed9c7c5 2024-11-21T11:14:49,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/f062497be3f94498b3b45d5a4eed80d6 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/f062497be3f94498b3b45d5a4eed80d6 2024-11-21T11:14:49,544 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/aa880840d4eb44bdb85983effe8f86b3 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/aa880840d4eb44bdb85983effe8f86b3 2024-11-21T11:14:49,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/42102e60828f43a8935f025b596fa4e6, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/ff20ef03f31e4225aa231e07f1c40422, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/aa8ccd886e674201ba62b442c463f505, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/84a1a0c23f9147b287a7801daf152fc4, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/2c15022308e3474baa8038f9a3efd5a6, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/aed09e60274243b2a257f6d112a6534d, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/d73949bcd52a4500ba8ce8f0f0f41217, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/cf3e8570320747bab12b770692e4e1c4, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/b735e166509d4d2891b2ddc1a6aa3baf, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/33a3a64823a34fef82311c3164d154b5, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/0a2218f2f5754b8281e9a0a28b4b5622, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/bb262f01070c4c268b052b613972475d, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/a6df1aab4af84d48975f29771c7309a8, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/02b3c9440844466a800778e818203568, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/bea260807f45457394bb350def2448e9, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/449f62f51f3f448db9e2c1d076e7cde7, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/358e7580168442d4ab4f1eec0430a0bd, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/d1da2b83f81741418952eab3fdd0aa05, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/836eea3068f1414c847a117940ced7e4, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/b2910e0267334c0ea37be0d9ff0f8984, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/8879f3f0ca54427e8a4f84e8680dda16, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/28a6082798a7424ca83419b3b041b330, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/e635d8417b0741499484510cebff3f9a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/7a34158cf44f4b9fbf3b768811c32a57, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/a8d4e223c30646de8954f6b6f871e78c, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/d58507e821454058adc559b0758bbff2, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/ab295362759646a186cd85b515dd2123, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/f82089aa5c664624bb699cf136f133cb, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/c579e3234b8c4d2b84fa074ce8102469, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/b6556184e5c04777b741176369fd1b10, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/ab04bdaa50aa4f7ab1176e28b423ff39] to archive 2024-11-21T11:14:49,547 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T11:14:49,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/42102e60828f43a8935f025b596fa4e6 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/42102e60828f43a8935f025b596fa4e6 2024-11-21T11:14:49,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/ff20ef03f31e4225aa231e07f1c40422 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/ff20ef03f31e4225aa231e07f1c40422 2024-11-21T11:14:49,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/aa8ccd886e674201ba62b442c463f505 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/aa8ccd886e674201ba62b442c463f505 2024-11-21T11:14:49,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/84a1a0c23f9147b287a7801daf152fc4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/84a1a0c23f9147b287a7801daf152fc4 2024-11-21T11:14:49,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/2c15022308e3474baa8038f9a3efd5a6 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/2c15022308e3474baa8038f9a3efd5a6 2024-11-21T11:14:49,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/aed09e60274243b2a257f6d112a6534d to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/aed09e60274243b2a257f6d112a6534d 2024-11-21T11:14:49,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/d73949bcd52a4500ba8ce8f0f0f41217 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/d73949bcd52a4500ba8ce8f0f0f41217 2024-11-21T11:14:49,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/cf3e8570320747bab12b770692e4e1c4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/cf3e8570320747bab12b770692e4e1c4 2024-11-21T11:14:49,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/b735e166509d4d2891b2ddc1a6aa3baf to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/b735e166509d4d2891b2ddc1a6aa3baf 2024-11-21T11:14:49,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/33a3a64823a34fef82311c3164d154b5 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/33a3a64823a34fef82311c3164d154b5 2024-11-21T11:14:49,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/0a2218f2f5754b8281e9a0a28b4b5622 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/0a2218f2f5754b8281e9a0a28b4b5622 2024-11-21T11:14:49,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/bb262f01070c4c268b052b613972475d to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/bb262f01070c4c268b052b613972475d 2024-11-21T11:14:49,564 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/a6df1aab4af84d48975f29771c7309a8 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/a6df1aab4af84d48975f29771c7309a8 2024-11-21T11:14:49,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/02b3c9440844466a800778e818203568 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/02b3c9440844466a800778e818203568 2024-11-21T11:14:49,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/bea260807f45457394bb350def2448e9 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/bea260807f45457394bb350def2448e9 2024-11-21T11:14:49,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/449f62f51f3f448db9e2c1d076e7cde7 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/449f62f51f3f448db9e2c1d076e7cde7 2024-11-21T11:14:49,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/358e7580168442d4ab4f1eec0430a0bd to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/358e7580168442d4ab4f1eec0430a0bd 2024-11-21T11:14:49,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/d1da2b83f81741418952eab3fdd0aa05 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/d1da2b83f81741418952eab3fdd0aa05 2024-11-21T11:14:49,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/836eea3068f1414c847a117940ced7e4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/836eea3068f1414c847a117940ced7e4 2024-11-21T11:14:49,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/b2910e0267334c0ea37be0d9ff0f8984 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/b2910e0267334c0ea37be0d9ff0f8984 2024-11-21T11:14:49,574 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/8879f3f0ca54427e8a4f84e8680dda16 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/8879f3f0ca54427e8a4f84e8680dda16 2024-11-21T11:14:49,576 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/28a6082798a7424ca83419b3b041b330 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/28a6082798a7424ca83419b3b041b330 2024-11-21T11:14:49,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/e635d8417b0741499484510cebff3f9a to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/e635d8417b0741499484510cebff3f9a 2024-11-21T11:14:49,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/7a34158cf44f4b9fbf3b768811c32a57 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/7a34158cf44f4b9fbf3b768811c32a57 2024-11-21T11:14:49,579 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/a8d4e223c30646de8954f6b6f871e78c to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/a8d4e223c30646de8954f6b6f871e78c 2024-11-21T11:14:49,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/d58507e821454058adc559b0758bbff2 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/d58507e821454058adc559b0758bbff2 2024-11-21T11:14:49,582 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/ab295362759646a186cd85b515dd2123 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/ab295362759646a186cd85b515dd2123 2024-11-21T11:14:49,583 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/f82089aa5c664624bb699cf136f133cb to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/f82089aa5c664624bb699cf136f133cb 2024-11-21T11:14:49,585 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/c579e3234b8c4d2b84fa074ce8102469 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/c579e3234b8c4d2b84fa074ce8102469 2024-11-21T11:14:49,586 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/b6556184e5c04777b741176369fd1b10 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/b6556184e5c04777b741176369fd1b10 2024-11-21T11:14:49,587 DEBUG [StoreCloser-TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/ab04bdaa50aa4f7ab1176e28b423ff39 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/ab04bdaa50aa4f7ab1176e28b423ff39 2024-11-21T11:14:49,592 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/recovered.edits/520.seqid, newMaxSeqId=520, maxSeqId=4 2024-11-21T11:14:49,593 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. 2024-11-21T11:14:49,593 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1676): Region close journal for 960784f3d73443a7fc815210ff1818b4: Waiting for close lock at 1732187687774Running coprocessor pre-close hooks at 1732187687774Disabling compacts and flushes for region at 1732187687774Disabling writes for close at 1732187687774Obtaining lock to block concurrent updates at 1732187687774Preparing flush snapshotting stores in 960784f3d73443a7fc815210ff1818b4 at 1732187687774Finished memstore snapshotting TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1732187687775 (+1 ms)Flushing stores of TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4. at 1732187687775Flushing 960784f3d73443a7fc815210ff1818b4/A: creating writer at 1732187687775Flushing mob file 960784f3d73443a7fc815210ff1818b4/A: appending metadata at 1732187687785 (+10 ms)Flushing mob file 960784f3d73443a7fc815210ff1818b4/A: closing flushed file at 1732187687785Flushing 960784f3d73443a7fc815210ff1818b4/A: appending metadata at 1732187688198 (+413 ms)Flushing 960784f3d73443a7fc815210ff1818b4/A: closing flushed file at 1732187688198Flushing 960784f3d73443a7fc815210ff1818b4/B: creating writer at 1732187688606 (+408 ms)Flushing 960784f3d73443a7fc815210ff1818b4/B: appending metadata at 1732187688609 (+3 ms)Flushing 960784f3d73443a7fc815210ff1818b4/B: closing flushed file at 1732187688609Flushing 960784f3d73443a7fc815210ff1818b4/C: creating writer at 1732187689018 (+409 ms)Flushing 960784f3d73443a7fc815210ff1818b4/C: appending metadata at 1732187689020 (+2 ms)Flushing 960784f3d73443a7fc815210ff1818b4/C: closing flushed file at 1732187689020Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@803faf: reopening flushed file at 1732187689430 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4175795: reopening flushed file at 1732187689440 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a34b838: reopening flushed file at 1732187689446 (+6 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 960784f3d73443a7fc815210ff1818b4 in 1678ms, sequenceid=517, compaction requested=true at 1732187689452 (+6 ms)Writing region close event to WAL at 1732187689589 (+137 ms)Running coprocessor post-close hooks at 1732187689593 (+4 ms)Closed at 1732187689593 2024-11-21T11:14:49,596 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(157): Closed 960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:49,596 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=960784f3d73443a7fc815210ff1818b4, regionState=CLOSED 2024-11-21T11:14:49,597 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure 960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 because future has completed 2024-11-21T11:14:49,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-21T11:14:49,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseRegionProcedure 960784f3d73443a7fc815210ff1818b4, server=7b462513bfc2,33417,1732187618276 in 1.9780 sec 2024-11-21T11:14:49,601 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-21T11:14:49,601 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=960784f3d73443a7fc815210ff1818b4, UNASSIGN in 1.9840 sec 2024-11-21T11:14:49,604 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-21T11:14:49,604 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9880 sec 2024-11-21T11:14:49,606 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732187689605"}]},"ts":"1732187689605"} 2024-11-21T11:14:49,607 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T11:14:49,607 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T11:14:49,608 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9980 sec 2024-11-21T11:14:49,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-21T11:14:49,756 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:14:49,756 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T11:14:49,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T11:14:49,758 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T11:14:49,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T11:14:49,759 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=64, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T11:14:49,764 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:49,766 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A, FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B, FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C, FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/recovered.edits] 2024-11-21T11:14:49,770 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/57ee4acccc0d4b50925f2528a3ae299f to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/57ee4acccc0d4b50925f2528a3ae299f 2024-11-21T11:14:49,771 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/69864eb816b34092a86056f7dac92923 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/69864eb816b34092a86056f7dac92923 2024-11-21T11:14:49,773 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/f281bdcf6bd748dd9523bde9f3b776cd to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/A/f281bdcf6bd748dd9523bde9f3b776cd 2024-11-21T11:14:49,775 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/13ac38f3cd7a4bd884d59faddda48105 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/13ac38f3cd7a4bd884d59faddda48105 2024-11-21T11:14:49,776 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/3e8293810a8b4dae93a6873e42184436 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/3e8293810a8b4dae93a6873e42184436 2024-11-21T11:14:49,778 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/9608e3cea86a49be862c58071bca4c4e to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/B/9608e3cea86a49be862c58071bca4c4e 2024-11-21T11:14:49,781 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/31041caab1cf497298515371f0716766 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/31041caab1cf497298515371f0716766 2024-11-21T11:14:49,782 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/4f01ef00b5094af8ac444d1f7fb1a691 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/4f01ef00b5094af8ac444d1f7fb1a691 2024-11-21T11:14:49,784 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/7b2594b2623a477ca0ae2a1fd16d95de to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/C/7b2594b2623a477ca0ae2a1fd16d95de 2024-11-21T11:14:49,788 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/recovered.edits/520.seqid to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4/recovered.edits/520.seqid 2024-11-21T11:14:49,788 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:49,788 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T11:14:49,789 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T11:14:49,790 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-21T11:14:49,794 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112107c86f81f69b46508680bbab79737037_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112107c86f81f69b46508680bbab79737037_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:49,796 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411210c3d116136ed47559170e389e924e86f_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411210c3d116136ed47559170e389e924e86f_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:49,797 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213b210e9151094a088422d5dc1cc28b3d_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213b210e9151094a088422d5dc1cc28b3d_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:49,798 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214f11fe6d28e74f62b364b28416de2bdf_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214f11fe6d28e74f62b364b28416de2bdf_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:49,800 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215182e3c9c24140be849170758c3210da_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215182e3c9c24140be849170758c3210da_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:49,801 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215290bd3d45904f1297657d2ff9d27b47_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215290bd3d45904f1297657d2ff9d27b47_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:49,803 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112152d29e7c17634a68a1856a9efbe237cd_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112152d29e7c17634a68a1856a9efbe237cd_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:49,804 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121654de44d62104e1082a69700f67e1ee2_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121654de44d62104e1082a69700f67e1ee2_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:49,806 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112169f23f98e538415cbc7f42d9d6daf483_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112169f23f98e538415cbc7f42d9d6daf483_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:49,807 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112171e7eb530ed04ad28df7fe2040c7ed92_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112171e7eb530ed04ad28df7fe2040c7ed92_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:49,809 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121846e8aceff0f4c9b8adb756de188fe4c_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121846e8aceff0f4c9b8adb756de188fe4c_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:49,811 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112192f327aeb7134c4e95856c49b4a23f67_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112192f327aeb7134c4e95856c49b4a23f67_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:49,812 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219e25d317eaba4744a6e8456516a3c0fa_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219e25d317eaba4744a6e8456516a3c0fa_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:49,814 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b3c4a964efb341908e5ed4503a4368d1_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b3c4a964efb341908e5ed4503a4368d1_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:49,815 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b643a4a94ee0423cb05c8633c2e132b1_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b643a4a94ee0423cb05c8633c2e132b1_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:49,817 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b96a20c796d041ad8f8f463ae76dcd7f_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b96a20c796d041ad8f8f463ae76dcd7f_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:49,818 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121bd571334ef684175a692e19f4617885f_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121bd571334ef684175a692e19f4617885f_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:49,819 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c23c76f6ea22409dac96cfd87c314f77_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c23c76f6ea22409dac96cfd87c314f77_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:49,821 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121cb7084c83b404433ba811b52e087bc84_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121cb7084c83b404433ba811b52e087bc84_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:49,822 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e486c3499f4e40af81f4cc292e821f0f_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e486c3499f4e40af81f4cc292e821f0f_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:49,823 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121eed1378065b14199ba543bc7a7e2d35f_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121eed1378065b14199ba543bc7a7e2d35f_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:49,824 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f99a629553d647789d0d06956188e114_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f99a629553d647789d0d06956188e114_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:49,825 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f9cc7af5667044dc81efe468b49b7224_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f9cc7af5667044dc81efe468b49b7224_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:49,826 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121fd755ec36f3344f39e45beece4d62c00_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121fd755ec36f3344f39e45beece4d62c00_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:49,828 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121fda071f51c61468fbd37f8edd517a6ad_960784f3d73443a7fc815210ff1818b4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121fda071f51c61468fbd37f8edd517a6ad_960784f3d73443a7fc815210ff1818b4 2024-11-21T11:14:49,829 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T11:14:49,831 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=64, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T11:14:49,834 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T11:14:49,836 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T11:14:49,838 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=64, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T11:14:49,838 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T11:14:49,838 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732187689838"}]},"ts":"9223372036854775807"} 2024-11-21T11:14:49,840 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T11:14:49,840 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 960784f3d73443a7fc815210ff1818b4, NAME => 'TestAcidGuarantees,,1732187654682.960784f3d73443a7fc815210ff1818b4.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T11:14:49,840 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T11:14:49,840 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732187689840"}]},"ts":"9223372036854775807"} 2024-11-21T11:14:49,842 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T11:14:49,843 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=64, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T11:14:49,844 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 87 msec 2024-11-21T11:14:49,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T11:14:49,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T11:14:49,866 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:14:49,876 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 247) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-672728481_22 at /127.0.0.1:53722 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c/cluster_eb393522-c18a-83a9-045f-e99da161695d/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c/cluster_eb393522-c18a-83a9-045f-e99da161695d/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=546 (was 395) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5402 (was 5834) 2024-11-21T11:14:49,885 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=546, ProcessCount=11, AvailableMemoryMB=5401 2024-11-21T11:14:49,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T11:14:49,887 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T11:14:49,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T11:14:49,889 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T11:14:49,889 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:49,889 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 65 2024-11-21T11:14:49,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-21T11:14:49,890 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T11:14:49,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742090_1266 (size=1061) 2024-11-21T11:14:49,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-21T11:14:50,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-21T11:14:50,299 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322 2024-11-21T11:14:50,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742091_1267 (size=53) 2024-11-21T11:14:50,314 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:50,315 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 42a6ff9198257177a61e80b1f764ee4c, disabling compactions & flushes 2024-11-21T11:14:50,315 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:50,315 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:50,315 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. after waiting 0 ms 2024-11-21T11:14:50,315 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:50,315 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:50,315 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 42a6ff9198257177a61e80b1f764ee4c: Waiting for close lock at 1732187690315Disabling compacts and flushes for region at 1732187690315Disabling writes for close at 1732187690315Writing region close event to WAL at 1732187690315Closed at 1732187690315 2024-11-21T11:14:50,316 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T11:14:50,316 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732187690316"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732187690316"}]},"ts":"1732187690316"} 2024-11-21T11:14:50,318 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T11:14:50,318 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T11:14:50,319 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732187690318"}]},"ts":"1732187690318"} 2024-11-21T11:14:50,320 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T11:14:50,320 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42a6ff9198257177a61e80b1f764ee4c, ASSIGN}] 2024-11-21T11:14:50,321 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42a6ff9198257177a61e80b1f764ee4c, ASSIGN 2024-11-21T11:14:50,322 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=42a6ff9198257177a61e80b1f764ee4c, ASSIGN; state=OFFLINE, location=7b462513bfc2,33417,1732187618276; forceNewPlan=false, retain=false 2024-11-21T11:14:50,473 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=42a6ff9198257177a61e80b1f764ee4c, regionState=OPENING, regionLocation=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:50,475 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=42a6ff9198257177a61e80b1f764ee4c, ASSIGN because future has completed 2024-11-21T11:14:50,475 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276}] 2024-11-21T11:14:50,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-21T11:14:50,630 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:50,630 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7752): Opening region: {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} 2024-11-21T11:14:50,631 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 42a6ff9198257177a61e80b1f764ee4c 2024-11-21T11:14:50,631 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:50,631 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7794): checking encryption for 42a6ff9198257177a61e80b1f764ee4c 2024-11-21T11:14:50,631 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7797): checking classloading for 42a6ff9198257177a61e80b1f764ee4c 2024-11-21T11:14:50,632 INFO [StoreOpener-42a6ff9198257177a61e80b1f764ee4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 42a6ff9198257177a61e80b1f764ee4c 2024-11-21T11:14:50,633 INFO [StoreOpener-42a6ff9198257177a61e80b1f764ee4c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T11:14:50,633 INFO [StoreOpener-42a6ff9198257177a61e80b1f764ee4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42a6ff9198257177a61e80b1f764ee4c columnFamilyName A 2024-11-21T11:14:50,633 DEBUG [StoreOpener-42a6ff9198257177a61e80b1f764ee4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:50,634 INFO [StoreOpener-42a6ff9198257177a61e80b1f764ee4c-1 {}] regionserver.HStore(327): Store=42a6ff9198257177a61e80b1f764ee4c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:50,634 INFO [StoreOpener-42a6ff9198257177a61e80b1f764ee4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 42a6ff9198257177a61e80b1f764ee4c 2024-11-21T11:14:50,635 INFO [StoreOpener-42a6ff9198257177a61e80b1f764ee4c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T11:14:50,635 INFO [StoreOpener-42a6ff9198257177a61e80b1f764ee4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42a6ff9198257177a61e80b1f764ee4c columnFamilyName B 2024-11-21T11:14:50,635 DEBUG [StoreOpener-42a6ff9198257177a61e80b1f764ee4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:50,635 INFO [StoreOpener-42a6ff9198257177a61e80b1f764ee4c-1 {}] regionserver.HStore(327): Store=42a6ff9198257177a61e80b1f764ee4c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:50,635 INFO [StoreOpener-42a6ff9198257177a61e80b1f764ee4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 42a6ff9198257177a61e80b1f764ee4c 2024-11-21T11:14:50,636 INFO [StoreOpener-42a6ff9198257177a61e80b1f764ee4c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T11:14:50,636 INFO [StoreOpener-42a6ff9198257177a61e80b1f764ee4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42a6ff9198257177a61e80b1f764ee4c columnFamilyName C 2024-11-21T11:14:50,636 DEBUG [StoreOpener-42a6ff9198257177a61e80b1f764ee4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:50,637 INFO [StoreOpener-42a6ff9198257177a61e80b1f764ee4c-1 {}] regionserver.HStore(327): Store=42a6ff9198257177a61e80b1f764ee4c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:50,637 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:50,637 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1038): replaying wal for 42a6ff9198257177a61e80b1f764ee4c 2024-11-21T11:14:50,638 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c 2024-11-21T11:14:50,638 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c 2024-11-21T11:14:50,639 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1048): stopping wal replay for 42a6ff9198257177a61e80b1f764ee4c 2024-11-21T11:14:50,639 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1060): Cleaning up temporary data for 42a6ff9198257177a61e80b1f764ee4c 2024-11-21T11:14:50,639 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T11:14:50,640 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1093): writing seq id for 42a6ff9198257177a61e80b1f764ee4c 2024-11-21T11:14:50,642 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T11:14:50,642 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1114): Opened 42a6ff9198257177a61e80b1f764ee4c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65032193, jitterRate=-0.030944809317588806}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T11:14:50,642 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 42a6ff9198257177a61e80b1f764ee4c 2024-11-21T11:14:50,643 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1006): Region open journal for 42a6ff9198257177a61e80b1f764ee4c: Running coprocessor pre-open hook at 1732187690631Writing region info on filesystem at 1732187690631Initializing all the Stores at 1732187690631Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187690632 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187690632Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187690632Cleaning up temporary data from old regions at 1732187690639 (+7 ms)Running coprocessor post-open hooks at 1732187690642 (+3 ms)Region opened successfully at 1732187690643 (+1 ms) 2024-11-21T11:14:50,644 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., pid=67, masterSystemTime=1732187690627 2024-11-21T11:14:50,645 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:50,645 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:50,646 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=42a6ff9198257177a61e80b1f764ee4c, regionState=OPEN, openSeqNum=2, regionLocation=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:50,647 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 because future has completed 2024-11-21T11:14:50,649 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-21T11:14:50,649 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure 42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 in 172 msec 2024-11-21T11:14:50,650 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-21T11:14:50,650 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42a6ff9198257177a61e80b1f764ee4c, ASSIGN in 329 msec 2024-11-21T11:14:50,651 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T11:14:50,651 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732187690651"}]},"ts":"1732187690651"} 2024-11-21T11:14:50,652 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T11:14:50,653 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T11:14:50,654 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 766 msec 2024-11-21T11:14:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-21T11:14:51,026 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:14:51,027 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe11fca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:51,027 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:14:51,027 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:14:51,028 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:14:51,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:14:51,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:14:51,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12e008d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:51,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:14:51,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:14:51,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:51,030 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57242, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:14:51,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@247fce51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:51,030 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:14:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:14:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:14:51,032 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51736, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:14:51,033 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T11:14:51,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 7b462513bfc2,36543,1732187617563 2024-11-21T11:14:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6d9ccb20 2024-11-21T11:14:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T11:14:51,034 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57246, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T11:14:51,035 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b5b273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:51,035 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:14:51,035 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:14:51,036 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:14:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:14:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:14:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cfa5c2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:14:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:14:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:51,037 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:14:51,037 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1783c2cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:51,038 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1240be3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:51,038 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:14:51,038 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:14:51,039 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:14:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:14:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:14:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@685c3cb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:14:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:14:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:51,040 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57290, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:14:51,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@378529e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:51,041 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:51,041 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:14:51,041 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:14:51,041 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:14:51,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:14:51,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:14:51,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a7dcbe6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:51,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:14:51,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:14:51,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:51,042 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:14:51,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bf2ebf3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:51,043 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:51,043 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:14:51,043 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:14:51,044 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:14:51,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:14:51,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:14:51,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4aed8c02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:51,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:14:51,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:14:51,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:51,045 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:14:51,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55919460, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:51,046 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128d97bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:51,046 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:14:51,046 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:14:51,046 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:14:51,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:14:51,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:14:51,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@441e8059, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:51,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:14:51,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:14:51,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:51,048 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:14:51,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22a91284, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:51,048 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626de388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:51,048 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:14:51,049 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:14:51,049 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:14:51,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:14:51,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:14:51,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@598bb1cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:51,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:14:51,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:14:51,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:51,050 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:14:51,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5100bdd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:51,051 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a5c877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:51,051 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:14:51,051 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:14:51,052 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:14:51,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:14:51,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:14:51,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59ed6409, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:51,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:14:51,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:14:51,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:51,053 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:14:51,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@117c7e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:51,054 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679f7426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:51,054 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:14:51,054 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:14:51,055 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:14:51,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:14:51,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:14:51,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21968ee0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:51,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:14:51,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:14:51,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:51,056 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:14:51,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73a9df9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:51,057 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b71af22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:51,057 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:14:51,057 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:14:51,057 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:14:51,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:14:51,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:14:51,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2631cb07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:51,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:14:51,058 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:14:51,058 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:51,058 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57418, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:14:51,059 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f4af035, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:51,059 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728d338, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:51,059 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:14:51,059 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:14:51,060 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:14:51,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:14:51,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:14:51,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b98a4f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:51,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:14:51,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:14:51,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:51,061 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:14:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@453ccae1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:51,062 DEBUG [Thread-1239 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:14:51,062 DEBUG [Thread-1232 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:14:51,062 DEBUG [Thread-1245 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:14:51,062 DEBUG [Thread-1241 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:14:51,062 DEBUG [Thread-1228 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:14:51,062 DEBUG [Thread-1247 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:14:51,062 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:14:51,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:14:51,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:14:51,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:14:51,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:14:51,063 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:14:51,063 DEBUG [Thread-1236 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:14:51,063 DEBUG [Thread-1230 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:14:51,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:14:51,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:14:51,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:14:51,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:14:51,063 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:14:51,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:14:51,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:14:51,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-21T11:14:51,064 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51750, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:14:51,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:14:51,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:14:51,064 DEBUG [Thread-1234 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:14:51,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:14:51,064 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:14:51,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:14:51,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T11:14:51,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:14:51,065 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:14:51,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:14:51,065 DEBUG [Thread-1243 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:14:51,066 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:14:51,066 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:14:51,066 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:14:51,066 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:14:51,066 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:14:51,066 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:14:51,066 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:14:51,067 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:14:51,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:14:51,067 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:14:51,067 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:14:51,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2] 2024-11-21T11:14:51,068 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:14:51,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2] 2024-11-21T11:14:51,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2] 2024-11-21T11:14:51,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2] 2024-11-21T11:14:51,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2] 2024-11-21T11:14:51,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2] 2024-11-21T11:14:51,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2] 2024-11-21T11:14:51,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2] 2024-11-21T11:14:51,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2] 2024-11-21T11:14:51,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2] 2024-11-21T11:14:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 42a6ff9198257177a61e80b1f764ee4c 2024-11-21T11:14:51,079 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42a6ff9198257177a61e80b1f764ee4c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T11:14:51,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=A 2024-11-21T11:14:51,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:51,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=B 2024-11-21T11:14:51,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:51,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=C 2024-11-21T11:14:51,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:51,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/21bb9a2481e64179bd77c70937f1cfbd is 50, key is test_row_0/A:col10/1732187691079/Put/seqid=0 2024-11-21T11:14:51,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:51,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51758 deadline: 1732187751109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:51,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732187691106, max timestamp=1732187691113Num uniques 859; 2024-11-21T11:14:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T11:14:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:51,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732187691106, max timestamp=1732187691113Num uniques 859; 2024-11-21T11:14:51,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T11:14:51,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732187691106, max timestamp=1732187691113Num uniques 859; 2024-11-21T11:14:51,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T11:14:51,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742092_1268 (size=12001) 2024-11-21T11:14:51,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/21bb9a2481e64179bd77c70937f1cfbd 2024-11-21T11:14:51,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/5e17ecaa63cc4e77b982d314014b7faf is 50, key is test_row_0/B:col10/1732187691079/Put/seqid=0 2024-11-21T11:14:51,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51744 deadline: 1732187751141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:51,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:51,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:51,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187751142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:51,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:51,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51768 deadline: 1732187751145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:51,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:51,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187751150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:51,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:51,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742093_1269 (size=12001) 2024-11-21T11:14:51,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/5e17ecaa63cc4e77b982d314014b7faf 2024-11-21T11:14:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T11:14:51,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/d9fddd036ec0486fa4d1db0d0a3d8266 is 50, key is test_row_0/C:col10/1732187691079/Put/seqid=0 2024-11-21T11:14:51,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742094_1270 (size=12001) 2024-11-21T11:14:51,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T11:14:51,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:51,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:14:51,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:51,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:51,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:51,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:51,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51758 deadline: 1732187751225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:51,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:51,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:51,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51744 deadline: 1732187751256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:51,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:51,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51768 deadline: 1732187751256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:51,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:51,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:51,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187751256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:51,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:51,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:51,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:51,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187751265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:51,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:51,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T11:14:51,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:51,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:14:51,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:51,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:51,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T11:14:51,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51758 deadline: 1732187751436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:51,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:51,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51744 deadline: 1732187751465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:51,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:51,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51768 deadline: 1732187751465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:51,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:51,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:51,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187751465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:51,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:51,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:51,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187751476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:51,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:51,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T11:14:51,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:51,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:14:51,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:51,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:51,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:51,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/d9fddd036ec0486fa4d1db0d0a3d8266 2024-11-21T11:14:51,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/21bb9a2481e64179bd77c70937f1cfbd as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/21bb9a2481e64179bd77c70937f1cfbd 2024-11-21T11:14:51,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/21bb9a2481e64179bd77c70937f1cfbd, entries=150, sequenceid=12, filesize=11.7 K 2024-11-21T11:14:51,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/5e17ecaa63cc4e77b982d314014b7faf as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/5e17ecaa63cc4e77b982d314014b7faf 2024-11-21T11:14:51,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/5e17ecaa63cc4e77b982d314014b7faf, entries=150, sequenceid=12, filesize=11.7 K 2024-11-21T11:14:51,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/d9fddd036ec0486fa4d1db0d0a3d8266 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/d9fddd036ec0486fa4d1db0d0a3d8266 2024-11-21T11:14:51,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/d9fddd036ec0486fa4d1db0d0a3d8266, entries=150, sequenceid=12, filesize=11.7 K 2024-11-21T11:14:51,634 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=129.71 KB/132820 for 42a6ff9198257177a61e80b1f764ee4c in 555ms, sequenceid=12, compaction requested=false 2024-11-21T11:14:51,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:14:51,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T11:14:51,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:51,680 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 42a6ff9198257177a61e80b1f764ee4c 3/3 column families, dataSize=129.71 KB heapSize=365.45 KB 2024-11-21T11:14:51,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=A 2024-11-21T11:14:51,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:14:51,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=B 2024-11-21T11:14:51,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:14:51,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=C 2024-11-21T11:14:51,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:14:51,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/18064a02d71b43bf8c00d4c39e142ebd is 50, key is test_row_0/A:col10/1732187691140/Put/seqid=0 2024-11-21T11:14:51,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742095_1271 (size=12001) 2024-11-21T11:14:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T11:14:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 42a6ff9198257177a61e80b1f764ee4c 2024-11-21T11:14:51,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:14:51,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187751775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:51,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:51,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51758 deadline: 1732187751779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:51,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:51,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:51,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51744 deadline: 1732187751780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51768 deadline: 1732187751780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:51,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:51,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:51,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187751785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:51,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:51,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:51,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51758 deadline: 1732187751895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51744 deadline: 1732187751896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:51,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:51,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51768 deadline: 1732187751896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:51,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:51,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:51,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:51,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:52,092 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/18064a02d71b43bf8c00d4c39e142ebd 2024-11-21T11:14:52,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:52,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51768 deadline: 1732187752108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:52,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:52,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:52,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:52,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:52,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51758 deadline: 1732187752112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:52,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:52,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:52,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:52,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:52,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51744 deadline: 1732187752112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:52,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:52,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:52,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:52,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/8c5e6139b53c42849e5ac1a98ef9aaf2 is 50, key is test_row_0/B:col10/1732187691140/Put/seqid=0 2024-11-21T11:14:52,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742096_1272 (size=12001) 2024-11-21T11:14:52,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T11:14:52,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:52,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187752285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:52,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:52,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:52,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:52,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:52,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187752295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:52,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:52,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:52,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:52,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:52,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51768 deadline: 1732187752415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:52,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:52,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:52,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:52,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:52,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51758 deadline: 1732187752425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:52,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:52,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:52,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:52,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:52,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51744 deadline: 1732187752425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:52,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:52,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:52,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:52,527 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/8c5e6139b53c42849e5ac1a98ef9aaf2 2024-11-21T11:14:52,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/5ffa0a59829a44c4ab2d7c257bf8d310 is 50, key is test_row_0/C:col10/1732187691140/Put/seqid=0 2024-11-21T11:14:52,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742097_1273 (size=12001) 2024-11-21T11:14:52,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:52,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51768 deadline: 1732187752925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:52,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:52,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:52,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:52,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:52,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51758 deadline: 1732187752935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:52,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:52,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51744 deadline: 1732187752935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:52,957 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/5ffa0a59829a44c4ab2d7c257bf8d310 2024-11-21T11:14:52,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/18064a02d71b43bf8c00d4c39e142ebd as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/18064a02d71b43bf8c00d4c39e142ebd 2024-11-21T11:14:52,971 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/18064a02d71b43bf8c00d4c39e142ebd, entries=150, sequenceid=47, filesize=11.7 K 2024-11-21T11:14:52,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/8c5e6139b53c42849e5ac1a98ef9aaf2 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/8c5e6139b53c42849e5ac1a98ef9aaf2 2024-11-21T11:14:52,977 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/8c5e6139b53c42849e5ac1a98ef9aaf2, entries=150, sequenceid=47, filesize=11.7 K 2024-11-21T11:14:52,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/5ffa0a59829a44c4ab2d7c257bf8d310 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/5ffa0a59829a44c4ab2d7c257bf8d310 2024-11-21T11:14:52,982 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/5ffa0a59829a44c4ab2d7c257bf8d310, entries=150, sequenceid=47, filesize=11.7 K 2024-11-21T11:14:52,983 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~365.41 KB/374176, currentSize=60.38 KB/61830 for 42a6ff9198257177a61e80b1f764ee4c in 1303ms, sequenceid=47, compaction requested=false 2024-11-21T11:14:52,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:14:52,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:52,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-21T11:14:52,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-21T11:14:52,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-21T11:14:52,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9180 sec 2024-11-21T11:14:52,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.9260 sec 2024-11-21T11:14:53,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T11:14:53,215 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:14:53,216 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:14:53,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-21T11:14:53,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T11:14:53,218 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:14:53,219 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:14:53,219 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:14:53,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 42a6ff9198257177a61e80b1f764ee4c 2024-11-21T11:14:53,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42a6ff9198257177a61e80b1f764ee4c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-21T11:14:53,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=A 2024-11-21T11:14:53,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:53,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=B 2024-11-21T11:14:53,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:53,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=C 2024-11-21T11:14:53,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:53,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/258caf68ffc04d6aa7d11f9166e4c6d2 is 50, key is test_row_0/A:col10/1732187691778/Put/seqid=0 2024-11-21T11:14:53,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742098_1274 (size=14341) 2024-11-21T11:14:53,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T11:14:53,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:53,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187753347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:53,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:53,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:53,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:53,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:53,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187753352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:53,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:53,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:53,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:53,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T11:14:53,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:53,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:14:53,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:53,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:53,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:53,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:53,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:53,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187753455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:53,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:53,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:53,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:53,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:53,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187753465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:53,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:53,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:53,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:53,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T11:14:53,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:53,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:14:53,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:53,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:53,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:53,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:53,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T11:14:53,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:53,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187753665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:53,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:53,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:53,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:53,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T11:14:53,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:53,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:14:53,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:53,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:53,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:53,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:53,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187753675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:53,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:53,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:53,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:53,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:53,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/258caf68ffc04d6aa7d11f9166e4c6d2 2024-11-21T11:14:53,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/eda8c547ea804e138cf9203e1b722f29 is 50, key is test_row_0/B:col10/1732187691778/Put/seqid=0 2024-11-21T11:14:53,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742099_1275 (size=12001) 2024-11-21T11:14:53,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T11:14:53,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:53,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:14:53,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:53,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:53,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:53,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:53,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T11:14:53,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:53,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51768 deadline: 1732187753935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:53,937 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_0' in 42a6ff9198257177a61e80b1f764ee4c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:14:53,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:53,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:53,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:53,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51758 deadline: 1732187753956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:53,958 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_0' in 42a6ff9198257177a61e80b1f764ee4c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:14:53,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:53,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:53,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:53,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:53,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51744 deadline: 1732187753956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:53,960 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_0' in 42a6ff9198257177a61e80b1f764ee4c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:14:53,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:53,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:53,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:53,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:53,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187753975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:53,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T11:14:53,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:53,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:14:53,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:53,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:53,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:53,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:53,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:53,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187753985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:53,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:53,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:53,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:54,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/eda8c547ea804e138cf9203e1b722f29 2024-11-21T11:14:54,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/7f823aaf74fa43d6929e71ba3ce27ef5 is 50, key is test_row_0/C:col10/1732187691778/Put/seqid=0 2024-11-21T11:14:54,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742100_1276 (size=12001) 2024-11-21T11:14:54,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T11:14:54,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:54,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:14:54,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:54,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:54,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:54,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:54,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T11:14:54,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:54,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:14:54,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:54,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:54,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:54,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:54,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T11:14:54,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T11:14:54,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:54,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:14:54,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:54,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:54,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:54,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:54,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187754485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:54,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:54,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:54,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:54,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187754495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:54,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:54,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:54,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:54,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/7f823aaf74fa43d6929e71ba3ce27ef5 2024-11-21T11:14:54,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/258caf68ffc04d6aa7d11f9166e4c6d2 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/258caf68ffc04d6aa7d11f9166e4c6d2 2024-11-21T11:14:54,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/258caf68ffc04d6aa7d11f9166e4c6d2, entries=200, sequenceid=60, filesize=14.0 K 2024-11-21T11:14:54,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/eda8c547ea804e138cf9203e1b722f29 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/eda8c547ea804e138cf9203e1b722f29 2024-11-21T11:14:54,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/eda8c547ea804e138cf9203e1b722f29, entries=150, sequenceid=60, filesize=11.7 K 2024-11-21T11:14:54,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/7f823aaf74fa43d6929e71ba3ce27ef5 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/7f823aaf74fa43d6929e71ba3ce27ef5 2024-11-21T11:14:54,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/7f823aaf74fa43d6929e71ba3ce27ef5, entries=150, sequenceid=60, filesize=11.7 K 2024-11-21T11:14:54,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 42a6ff9198257177a61e80b1f764ee4c in 1262ms, sequenceid=60, compaction requested=true 2024-11-21T11:14:54,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:14:54,562 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:14:54,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42a6ff9198257177a61e80b1f764ee4c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:14:54,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:54,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42a6ff9198257177a61e80b1f764ee4c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:14:54,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:54,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42a6ff9198257177a61e80b1f764ee4c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:14:54,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:14:54,562 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:14:54,563 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:14:54,563 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 42a6ff9198257177a61e80b1f764ee4c/A is initiating minor compaction (all files) 2024-11-21T11:14:54,563 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42a6ff9198257177a61e80b1f764ee4c/A in TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:54,563 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/21bb9a2481e64179bd77c70937f1cfbd, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/18064a02d71b43bf8c00d4c39e142ebd, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/258caf68ffc04d6aa7d11f9166e4c6d2] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp, totalSize=37.4 K 2024-11-21T11:14:54,563 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:14:54,563 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 42a6ff9198257177a61e80b1f764ee4c/B is initiating minor compaction (all files) 2024-11-21T11:14:54,563 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42a6ff9198257177a61e80b1f764ee4c/B in TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:54,563 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/5e17ecaa63cc4e77b982d314014b7faf, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/8c5e6139b53c42849e5ac1a98ef9aaf2, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/eda8c547ea804e138cf9203e1b722f29] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp, totalSize=35.2 K 2024-11-21T11:14:54,564 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e17ecaa63cc4e77b982d314014b7faf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732187691074 2024-11-21T11:14:54,564 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 21bb9a2481e64179bd77c70937f1cfbd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732187691074 2024-11-21T11:14:54,564 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 18064a02d71b43bf8c00d4c39e142ebd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732187691140 2024-11-21T11:14:54,564 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c5e6139b53c42849e5ac1a98ef9aaf2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732187691140 2024-11-21T11:14:54,565 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting eda8c547ea804e138cf9203e1b722f29, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732187691766 2024-11-21T11:14:54,565 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 258caf68ffc04d6aa7d11f9166e4c6d2, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732187691766 2024-11-21T11:14:54,575 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42a6ff9198257177a61e80b1f764ee4c#B#compaction#224 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:54,575 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42a6ff9198257177a61e80b1f764ee4c#A#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:54,576 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/899647521f59479ab443b14174494fa9 is 50, key is test_row_0/A:col10/1732187691778/Put/seqid=0 2024-11-21T11:14:54,576 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/86e1e6dc5bd04c3388414321802bac2f is 50, key is test_row_0/B:col10/1732187691778/Put/seqid=0 2024-11-21T11:14:54,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742101_1277 (size=12104) 2024-11-21T11:14:54,594 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/86e1e6dc5bd04c3388414321802bac2f as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/86e1e6dc5bd04c3388414321802bac2f 2024-11-21T11:14:54,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T11:14:54,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:54,598 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 42a6ff9198257177a61e80b1f764ee4c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T11:14:54,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=A 2024-11-21T11:14:54,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:54,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=B 2024-11-21T11:14:54,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:54,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=C 2024-11-21T11:14:54,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:54,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/c45f6a1f57504bd4a20c28127eb3a1ee is 50, key is test_row_0/A:col10/1732187693344/Put/seqid=0 2024-11-21T11:14:54,611 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42a6ff9198257177a61e80b1f764ee4c/B of 42a6ff9198257177a61e80b1f764ee4c into 86e1e6dc5bd04c3388414321802bac2f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:54,611 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:14:54,611 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., storeName=42a6ff9198257177a61e80b1f764ee4c/B, priority=13, startTime=1732187694562; duration=0sec 2024-11-21T11:14:54,612 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:14:54,612 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42a6ff9198257177a61e80b1f764ee4c:B 2024-11-21T11:14:54,612 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:14:54,613 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:14:54,613 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 42a6ff9198257177a61e80b1f764ee4c/C is initiating minor compaction (all files) 2024-11-21T11:14:54,613 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42a6ff9198257177a61e80b1f764ee4c/C in TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:54,613 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/d9fddd036ec0486fa4d1db0d0a3d8266, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/5ffa0a59829a44c4ab2d7c257bf8d310, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/7f823aaf74fa43d6929e71ba3ce27ef5] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp, totalSize=35.2 K 2024-11-21T11:14:54,614 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9fddd036ec0486fa4d1db0d0a3d8266, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732187691074 2024-11-21T11:14:54,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742102_1278 (size=12104) 2024-11-21T11:14:54,615 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ffa0a59829a44c4ab2d7c257bf8d310, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732187691140 2024-11-21T11:14:54,616 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f823aaf74fa43d6929e71ba3ce27ef5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732187691766 2024-11-21T11:14:54,622 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/899647521f59479ab443b14174494fa9 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/899647521f59479ab443b14174494fa9 2024-11-21T11:14:54,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742103_1279 (size=12001) 2024-11-21T11:14:54,625 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/c45f6a1f57504bd4a20c28127eb3a1ee 2024-11-21T11:14:54,628 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42a6ff9198257177a61e80b1f764ee4c#C#compaction#227 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:54,629 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/91c74615f89b460387683339ec635b47 is 50, key is test_row_0/C:col10/1732187691778/Put/seqid=0 2024-11-21T11:14:54,630 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42a6ff9198257177a61e80b1f764ee4c/A of 42a6ff9198257177a61e80b1f764ee4c into 899647521f59479ab443b14174494fa9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:54,630 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:14:54,630 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., storeName=42a6ff9198257177a61e80b1f764ee4c/A, priority=13, startTime=1732187694561; duration=0sec 2024-11-21T11:14:54,630 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:54,631 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42a6ff9198257177a61e80b1f764ee4c:A 2024-11-21T11:14:54,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742104_1280 (size=12104) 2024-11-21T11:14:54,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/391a591facfd43cabeb19347a0a7f1e4 is 50, key is test_row_0/B:col10/1732187693344/Put/seqid=0 2024-11-21T11:14:54,657 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/91c74615f89b460387683339ec635b47 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/91c74615f89b460387683339ec635b47 2024-11-21T11:14:54,664 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42a6ff9198257177a61e80b1f764ee4c/C of 42a6ff9198257177a61e80b1f764ee4c into 91c74615f89b460387683339ec635b47(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:54,664 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:14:54,664 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., storeName=42a6ff9198257177a61e80b1f764ee4c/C, priority=13, startTime=1732187694562; duration=0sec 2024-11-21T11:14:54,664 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:54,664 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42a6ff9198257177a61e80b1f764ee4c:C 2024-11-21T11:14:54,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742105_1281 (size=12001) 2024-11-21T11:14:54,674 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/391a591facfd43cabeb19347a0a7f1e4 2024-11-21T11:14:54,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/cb30b8a98bd84a3eb34cb9b17608efce is 50, key is test_row_0/C:col10/1732187693344/Put/seqid=0 2024-11-21T11:14:54,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742106_1282 (size=12001) 2024-11-21T11:14:54,711 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/cb30b8a98bd84a3eb34cb9b17608efce 2024-11-21T11:14:54,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/c45f6a1f57504bd4a20c28127eb3a1ee as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/c45f6a1f57504bd4a20c28127eb3a1ee 2024-11-21T11:14:54,721 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/c45f6a1f57504bd4a20c28127eb3a1ee, entries=150, sequenceid=79, filesize=11.7 K 2024-11-21T11:14:54,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/391a591facfd43cabeb19347a0a7f1e4 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/391a591facfd43cabeb19347a0a7f1e4 2024-11-21T11:14:54,727 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/391a591facfd43cabeb19347a0a7f1e4, entries=150, sequenceid=79, filesize=11.7 K 2024-11-21T11:14:54,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/cb30b8a98bd84a3eb34cb9b17608efce as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/cb30b8a98bd84a3eb34cb9b17608efce 2024-11-21T11:14:54,733 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/cb30b8a98bd84a3eb34cb9b17608efce, entries=150, sequenceid=79, filesize=11.7 K 2024-11-21T11:14:54,734 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 42a6ff9198257177a61e80b1f764ee4c in 136ms, sequenceid=79, compaction requested=false 2024-11-21T11:14:54,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:14:54,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:54,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-21T11:14:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-21T11:14:54,740 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-21T11:14:54,740 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5170 sec 2024-11-21T11:14:54,742 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.5240 sec 2024-11-21T11:14:55,118 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T11:14:55,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T11:14:55,365 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:14:55,366 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:14:55,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-21T11:14:55,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T11:14:55,368 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:14:55,369 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:14:55,369 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:14:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T11:14:55,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-21T11:14:55,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:55,522 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 42a6ff9198257177a61e80b1f764ee4c 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-21T11:14:55,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=A 2024-11-21T11:14:55,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:55,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=B 2024-11-21T11:14:55,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:55,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=C 2024-11-21T11:14:55,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:55,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 42a6ff9198257177a61e80b1f764ee4c 2024-11-21T11:14:55,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:14:55,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/d6060cdfd2e341bfb0f9d488cd9d41f9 is 50, key is test_row_0/A:col10/1732187695510/Put/seqid=0 2024-11-21T11:14:55,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742107_1283 (size=12001) 2024-11-21T11:14:55,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732187695554, max timestamp=1732187695574Num uniques 859; 2024-11-21T11:14:55,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732187695554, max timestamp=1732187695574Num uniques 859; 2024-11-21T11:14:55,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T11:14:55,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T11:14:55,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732187695554, max timestamp=1732187695574Num uniques 859; 2024-11-21T11:14:55,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T11:14:55,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732187695603, max timestamp=1732187695624Num uniques 1018; 2024-11-21T11:14:55,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-21T11:14:55,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732187695603, max timestamp=1732187695624Num uniques 1018; 2024-11-21T11:14:55,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-21T11:14:55,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732187695603, max timestamp=1732187695624Num uniques 1018; 2024-11-21T11:14:55,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-21T11:14:55,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187755673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:55,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:55,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:55,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:55,680 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732187695656, max timestamp=1732187695676Num uniques 1027; 2024-11-21T11:14:55,680 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-21T11:14:55,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732187695656, max timestamp=1732187695676Num uniques 1027; 2024-11-21T11:14:55,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-21T11:14:55,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732187695656, max timestamp=1732187695676Num uniques 1027; 2024-11-21T11:14:55,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-21T11:14:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T11:14:55,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732187695752, max timestamp=1732187695776Num uniques 1036; 2024-11-21T11:14:55,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-21T11:14:55,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732187695752, max timestamp=1732187695776Num uniques 1036; 2024-11-21T11:14:55,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-21T11:14:55,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732187695752, max timestamp=1732187695776Num uniques 1036; 2024-11-21T11:14:55,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-21T11:14:55,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732187695801, max timestamp=1732187695838Num uniques 1045; 2024-11-21T11:14:55,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-21T11:14:55,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732187695801, max timestamp=1732187695838Num uniques 1045; 2024-11-21T11:14:55,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-21T11:14:55,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732187695801, max timestamp=1732187695838Num uniques 1045; 2024-11-21T11:14:55,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-21T11:14:55,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187755882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:55,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:55,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:55,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:55,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187755886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:55,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:55,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:55,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:55,946 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/d6060cdfd2e341bfb0f9d488cd9d41f9 2024-11-21T11:14:55,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/e5cba1ebada74f72bf466d13299abfac is 50, key is test_row_0/B:col10/1732187695510/Put/seqid=0 2024-11-21T11:14:55,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51768 deadline: 1732187755955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:55,963 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_0' in 42a6ff9198257177a61e80b1f764ee4c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:14:55,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:55,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:55,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:55,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742108_1284 (size=12001) 2024-11-21T11:14:55,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51758 deadline: 1732187755975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:55,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51744 deadline: 1732187755975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:55,981 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_0' in 42a6ff9198257177a61e80b1f764ee4c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:14:55,981 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_0' in 42a6ff9198257177a61e80b1f764ee4c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:14:55,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:55,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:55,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:55,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:55,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:55,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T11:14:56,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187755995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:56,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:56,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:56,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187755995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:56,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:56,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:56,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:56,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:56,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:56,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187756206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:56,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:56,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:56,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187756206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:56,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:56,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:56,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:56,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:56,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:56,364 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/e5cba1ebada74f72bf466d13299abfac 2024-11-21T11:14:56,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/bb12bb4728c942fa842b39c6fd31dbbf is 50, key is test_row_0/C:col10/1732187695510/Put/seqid=0 2024-11-21T11:14:56,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742109_1285 (size=12001) 2024-11-21T11:14:56,375 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/bb12bb4728c942fa842b39c6fd31dbbf 2024-11-21T11:14:56,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/d6060cdfd2e341bfb0f9d488cd9d41f9 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/d6060cdfd2e341bfb0f9d488cd9d41f9 2024-11-21T11:14:56,386 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/d6060cdfd2e341bfb0f9d488cd9d41f9, entries=150, sequenceid=91, filesize=11.7 K 2024-11-21T11:14:56,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/e5cba1ebada74f72bf466d13299abfac as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/e5cba1ebada74f72bf466d13299abfac 2024-11-21T11:14:56,393 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/e5cba1ebada74f72bf466d13299abfac, entries=150, sequenceid=91, filesize=11.7 K 2024-11-21T11:14:56,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/bb12bb4728c942fa842b39c6fd31dbbf as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/bb12bb4728c942fa842b39c6fd31dbbf 2024-11-21T11:14:56,398 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/bb12bb4728c942fa842b39c6fd31dbbf, entries=150, sequenceid=91, filesize=11.7 K 2024-11-21T11:14:56,399 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 42a6ff9198257177a61e80b1f764ee4c in 877ms, sequenceid=91, compaction requested=true 2024-11-21T11:14:56,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:14:56,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:56,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-21T11:14:56,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-21T11:14:56,403 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-21T11:14:56,403 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0320 sec 2024-11-21T11:14:56,405 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.0380 sec 2024-11-21T11:14:56,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T11:14:56,505 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:14:56,506 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:14:56,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-21T11:14:56,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T11:14:56,507 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:14:56,508 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:14:56,508 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:14:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 42a6ff9198257177a61e80b1f764ee4c 2024-11-21T11:14:56,519 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42a6ff9198257177a61e80b1f764ee4c 3/3 column families, dataSize=147.60 KB heapSize=416.77 KB 2024-11-21T11:14:56,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=A 2024-11-21T11:14:56,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:14:56,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=B 2024-11-21T11:14:56,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:14:56,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=C 2024-11-21T11:14:56,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:14:56,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/15c92aa29d0c47349202b5f9b5f422d4 is 50, key is test_row_0/A:col10/1732187695868/Put/seqid=0 2024-11-21T11:14:56,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742110_1286 (size=14541) 2024-11-21T11:14:56,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:56,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187756535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:56,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:56,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:56,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:56,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:56,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187756536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:56,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:56,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:56,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:56,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T11:14:56,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:56,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187756646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:56,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:56,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:56,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:56,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:56,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187756646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:56,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:56,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:56,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:56,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T11:14:56,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:56,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:14:56,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:56,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:56,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:56,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T11:14:56,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:56,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:14:56,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:56,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:56,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:56,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:56,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T11:14:56,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:56,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187756855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:56,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:56,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187756855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:56,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:56,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:56,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:56,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:56,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:56,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:56,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/15c92aa29d0c47349202b5f9b5f422d4 2024-11-21T11:14:56,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/a98fdba762074a669c08d3448ef297f4 is 50, key is test_row_0/B:col10/1732187695868/Put/seqid=0 2024-11-21T11:14:56,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742111_1287 (size=12151) 2024-11-21T11:14:56,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T11:14:56,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:56,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:14:56,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:56,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:56,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:57,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T11:14:57,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:57,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:14:57,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:57,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:57,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T11:14:57,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187757165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:57,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:57,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187757165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:57,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:57,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:57,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:57,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:57,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:57,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T11:14:57,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:57,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:14:57,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:57,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:57,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:57,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/a98fdba762074a669c08d3448ef297f4 2024-11-21T11:14:57,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/40f84aaca53b48e5a92b0ec0cfdacaaf is 50, key is test_row_0/C:col10/1732187695868/Put/seqid=0 2024-11-21T11:14:57,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742112_1288 (size=12151) 2024-11-21T11:14:57,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T11:14:57,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:57,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:14:57,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:57,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:57,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:57,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T11:14:57,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:57,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:14:57,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:57,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:57,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:57,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:57,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T11:14:57,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:57,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:57,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187757675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:57,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187757676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:57,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:57,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:57,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:57,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:57,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:57,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:57,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T11:14:57,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:57,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:14:57,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:57,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:57,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:57,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:57,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/40f84aaca53b48e5a92b0ec0cfdacaaf 2024-11-21T11:14:57,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/15c92aa29d0c47349202b5f9b5f422d4 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/15c92aa29d0c47349202b5f9b5f422d4 2024-11-21T11:14:57,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/15c92aa29d0c47349202b5f9b5f422d4, entries=200, sequenceid=197, filesize=14.2 K 2024-11-21T11:14:57,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/a98fdba762074a669c08d3448ef297f4 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/a98fdba762074a669c08d3448ef297f4 2024-11-21T11:14:57,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/a98fdba762074a669c08d3448ef297f4, entries=150, sequenceid=197, filesize=11.9 K 2024-11-21T11:14:57,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/40f84aaca53b48e5a92b0ec0cfdacaaf as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/40f84aaca53b48e5a92b0ec0cfdacaaf 2024-11-21T11:14:57,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/40f84aaca53b48e5a92b0ec0cfdacaaf, entries=150, sequenceid=197, filesize=11.9 K 2024-11-21T11:14:57,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~416.72 KB/426720, currentSize=33.54 KB/34350 for 42a6ff9198257177a61e80b1f764ee4c in 1265ms, sequenceid=197, compaction requested=true 2024-11-21T11:14:57,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:14:57,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42a6ff9198257177a61e80b1f764ee4c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:14:57,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:57,784 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T11:14:57,784 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T11:14:57,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42a6ff9198257177a61e80b1f764ee4c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:14:57,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:57,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42a6ff9198257177a61e80b1f764ee4c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:14:57,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:14:57,787 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T11:14:57,787 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 42a6ff9198257177a61e80b1f764ee4c/B is initiating minor compaction (all files) 2024-11-21T11:14:57,787 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42a6ff9198257177a61e80b1f764ee4c/B in TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:57,787 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/86e1e6dc5bd04c3388414321802bac2f, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/391a591facfd43cabeb19347a0a7f1e4, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/e5cba1ebada74f72bf466d13299abfac, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/a98fdba762074a669c08d3448ef297f4] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp, totalSize=47.1 K 2024-11-21T11:14:57,788 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50647 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T11:14:57,788 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 42a6ff9198257177a61e80b1f764ee4c/A is initiating minor compaction (all files) 2024-11-21T11:14:57,788 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42a6ff9198257177a61e80b1f764ee4c/A in TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:57,788 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/899647521f59479ab443b14174494fa9, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/c45f6a1f57504bd4a20c28127eb3a1ee, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/d6060cdfd2e341bfb0f9d488cd9d41f9, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/15c92aa29d0c47349202b5f9b5f422d4] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp, totalSize=49.5 K 2024-11-21T11:14:57,788 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 899647521f59479ab443b14174494fa9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732187691766 2024-11-21T11:14:57,789 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 86e1e6dc5bd04c3388414321802bac2f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732187691766 2024-11-21T11:14:57,789 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting c45f6a1f57504bd4a20c28127eb3a1ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732187693344 2024-11-21T11:14:57,789 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 391a591facfd43cabeb19347a0a7f1e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732187693344 2024-11-21T11:14:57,790 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6060cdfd2e341bfb0f9d488cd9d41f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732187695509 2024-11-21T11:14:57,790 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting e5cba1ebada74f72bf466d13299abfac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732187695509 2024-11-21T11:14:57,790 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting a98fdba762074a669c08d3448ef297f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732187695868 2024-11-21T11:14:57,790 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15c92aa29d0c47349202b5f9b5f422d4, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732187695868 2024-11-21T11:14:57,803 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42a6ff9198257177a61e80b1f764ee4c#A#compaction#236 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:57,803 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42a6ff9198257177a61e80b1f764ee4c#B#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:57,803 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/c3b1ea763acf4d68bdb8fbeea0435f7c is 50, key is test_row_0/A:col10/1732187695868/Put/seqid=0 2024-11-21T11:14:57,803 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/d56be3c28620423f8009068c723242d0 is 50, key is test_row_0/B:col10/1732187695868/Put/seqid=0 2024-11-21T11:14:57,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742113_1289 (size=12391) 2024-11-21T11:14:57,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742114_1290 (size=12391) 2024-11-21T11:14:57,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T11:14:57,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:57,900 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 42a6ff9198257177a61e80b1f764ee4c 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-21T11:14:57,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=A 2024-11-21T11:14:57,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:57,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=B 2024-11-21T11:14:57,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:57,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=C 2024-11-21T11:14:57,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:57,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/8a0d11d9c5c8450f9cc0799d1509123e is 50, key is test_row_0/A:col10/1732187696532/Put/seqid=0 2024-11-21T11:14:57,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742115_1291 (size=12151) 2024-11-21T11:14:57,915 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/8a0d11d9c5c8450f9cc0799d1509123e 2024-11-21T11:14:57,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/348899dbf6aa4498a8bd15b0cebefc9b is 50, key is test_row_0/B:col10/1732187696532/Put/seqid=0 2024-11-21T11:14:57,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742116_1292 (size=12151) 2024-11-21T11:14:58,225 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/c3b1ea763acf4d68bdb8fbeea0435f7c as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/c3b1ea763acf4d68bdb8fbeea0435f7c 2024-11-21T11:14:58,230 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 42a6ff9198257177a61e80b1f764ee4c/A of 42a6ff9198257177a61e80b1f764ee4c into c3b1ea763acf4d68bdb8fbeea0435f7c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:58,230 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:14:58,230 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., storeName=42a6ff9198257177a61e80b1f764ee4c/A, priority=12, startTime=1732187697784; duration=0sec 2024-11-21T11:14:58,230 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:14:58,231 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42a6ff9198257177a61e80b1f764ee4c:A 2024-11-21T11:14:58,231 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T11:14:58,232 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T11:14:58,232 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 42a6ff9198257177a61e80b1f764ee4c/C is initiating minor compaction (all files) 2024-11-21T11:14:58,232 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42a6ff9198257177a61e80b1f764ee4c/C in TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:58,233 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/91c74615f89b460387683339ec635b47, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/cb30b8a98bd84a3eb34cb9b17608efce, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/bb12bb4728c942fa842b39c6fd31dbbf, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/40f84aaca53b48e5a92b0ec0cfdacaaf] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp, totalSize=47.1 K 2024-11-21T11:14:58,233 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91c74615f89b460387683339ec635b47, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732187691766 2024-11-21T11:14:58,233 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb30b8a98bd84a3eb34cb9b17608efce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732187693344 2024-11-21T11:14:58,234 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb12bb4728c942fa842b39c6fd31dbbf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732187695509 2024-11-21T11:14:58,234 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40f84aaca53b48e5a92b0ec0cfdacaaf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732187695868 2024-11-21T11:14:58,240 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/d56be3c28620423f8009068c723242d0 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/d56be3c28620423f8009068c723242d0 2024-11-21T11:14:58,246 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 42a6ff9198257177a61e80b1f764ee4c/B of 42a6ff9198257177a61e80b1f764ee4c into d56be3c28620423f8009068c723242d0(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:58,246 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:14:58,246 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., storeName=42a6ff9198257177a61e80b1f764ee4c/B, priority=12, startTime=1732187697784; duration=0sec 2024-11-21T11:14:58,247 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:58,247 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42a6ff9198257177a61e80b1f764ee4c:B 2024-11-21T11:14:58,251 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42a6ff9198257177a61e80b1f764ee4c#C#compaction#240 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:58,252 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/cdf2fbba64be4cbeac57f277db4eb00d is 50, key is test_row_0/C:col10/1732187695868/Put/seqid=0 2024-11-21T11:14:58,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742117_1293 (size=12391) 2024-11-21T11:14:58,338 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/348899dbf6aa4498a8bd15b0cebefc9b 2024-11-21T11:14:58,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/f30f9add9e1048f1b49bc448e9cb1d0c is 50, key is test_row_0/C:col10/1732187696532/Put/seqid=0 2024-11-21T11:14:58,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742118_1294 (size=12151) 2024-11-21T11:14:58,359 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/f30f9add9e1048f1b49bc448e9cb1d0c 2024-11-21T11:14:58,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/8a0d11d9c5c8450f9cc0799d1509123e as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/8a0d11d9c5c8450f9cc0799d1509123e 2024-11-21T11:14:58,372 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/8a0d11d9c5c8450f9cc0799d1509123e, entries=150, sequenceid=205, filesize=11.9 K 2024-11-21T11:14:58,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/348899dbf6aa4498a8bd15b0cebefc9b as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/348899dbf6aa4498a8bd15b0cebefc9b 2024-11-21T11:14:58,377 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/348899dbf6aa4498a8bd15b0cebefc9b, entries=150, sequenceid=205, filesize=11.9 K 2024-11-21T11:14:58,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/f30f9add9e1048f1b49bc448e9cb1d0c as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/f30f9add9e1048f1b49bc448e9cb1d0c 2024-11-21T11:14:58,392 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/f30f9add9e1048f1b49bc448e9cb1d0c, entries=150, sequenceid=205, filesize=11.9 K 2024-11-21T11:14:58,393 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 42a6ff9198257177a61e80b1f764ee4c in 493ms, sequenceid=205, compaction requested=false 2024-11-21T11:14:58,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:14:58,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:58,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-21T11:14:58,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-21T11:14:58,399 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-21T11:14:58,399 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8870 sec 2024-11-21T11:14:58,401 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.8940 sec 2024-11-21T11:14:58,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T11:14:58,656 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:14:58,656 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:14:58,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-21T11:14:58,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T11:14:58,658 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:14:58,659 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:14:58,659 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:14:58,672 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/cdf2fbba64be4cbeac57f277db4eb00d as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/cdf2fbba64be4cbeac57f277db4eb00d 2024-11-21T11:14:58,688 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 42a6ff9198257177a61e80b1f764ee4c/C of 42a6ff9198257177a61e80b1f764ee4c into cdf2fbba64be4cbeac57f277db4eb00d(size=12.1 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:58,689 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:14:58,689 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., storeName=42a6ff9198257177a61e80b1f764ee4c/C, priority=12, startTime=1732187697785; duration=0sec 2024-11-21T11:14:58,689 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:58,690 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42a6ff9198257177a61e80b1f764ee4c:C 2024-11-21T11:14:58,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 42a6ff9198257177a61e80b1f764ee4c 2024-11-21T11:14:58,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42a6ff9198257177a61e80b1f764ee4c 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-21T11:14:58,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=A 2024-11-21T11:14:58,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:58,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=B 2024-11-21T11:14:58,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:58,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=C 2024-11-21T11:14:58,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:14:58,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/b86ab614a948444ebcf0726d460aecec is 50, key is test_row_1/A:col10/1732187698700/Put/seqid=0 2024-11-21T11:14:58,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742119_1295 (size=9757) 2024-11-21T11:14:58,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732187698721, max timestamp=1732187698746Num uniques 859; 2024-11-21T11:14:58,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732187698721, max timestamp=1732187698746Num uniques 859; 2024-11-21T11:14:58,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T11:14:58,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T11:14:58,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732187698721, max timestamp=1732187698746Num uniques 859; 2024-11-21T11:14:58,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T11:14:58,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T11:14:58,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:58,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187758784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:58,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:58,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187758784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:58,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:58,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:58,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:58,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:58,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:58,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:58,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-21T11:14:58,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:58,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:14:58,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:58,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:58,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:58,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:58,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:58,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187758896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:58,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:58,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:58,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187758896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:58,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:58,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:58,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:58,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:58,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:58,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-21T11:14:58,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:58,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:14:58,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:58,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:58,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:58,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:58,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T11:14:59,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:59,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187759106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:59,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:59,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187759106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:59,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-21T11:14:59,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:59,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:14:59,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:59,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:59,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:59,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:59,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/b86ab614a948444ebcf0726d460aecec 2024-11-21T11:14:59,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/76a6b6185488406aa8484039b1173020 is 50, key is test_row_1/B:col10/1732187698700/Put/seqid=0 2024-11-21T11:14:59,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742120_1296 (size=9757) 2024-11-21T11:14:59,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-21T11:14:59,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:59,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:14:59,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:59,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:59,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:59,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:59,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T11:14:59,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:59,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187759415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:59,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:59,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:59,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187759415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:59,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:59,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:59,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:59,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:59,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:59,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-21T11:14:59,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:59,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:14:59,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:59,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:59,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:59,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:59,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/76a6b6185488406aa8484039b1173020 2024-11-21T11:14:59,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/44ee115bf1ec4d7e99e1438a02b3a677 is 50, key is test_row_1/C:col10/1732187698700/Put/seqid=0 2024-11-21T11:14:59,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742121_1297 (size=9757) 2024-11-21T11:14:59,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/44ee115bf1ec4d7e99e1438a02b3a677 2024-11-21T11:14:59,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/b86ab614a948444ebcf0726d460aecec as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/b86ab614a948444ebcf0726d460aecec 2024-11-21T11:14:59,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/b86ab614a948444ebcf0726d460aecec, entries=100, sequenceid=219, filesize=9.5 K 2024-11-21T11:14:59,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/76a6b6185488406aa8484039b1173020 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/76a6b6185488406aa8484039b1173020 2024-11-21T11:14:59,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/76a6b6185488406aa8484039b1173020, entries=100, sequenceid=219, filesize=9.5 K 2024-11-21T11:14:59,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/44ee115bf1ec4d7e99e1438a02b3a677 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/44ee115bf1ec4d7e99e1438a02b3a677 2024-11-21T11:14:59,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/44ee115bf1ec4d7e99e1438a02b3a677, entries=100, sequenceid=219, filesize=9.5 K 2024-11-21T11:14:59,575 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 42a6ff9198257177a61e80b1f764ee4c in 873ms, sequenceid=219, compaction requested=true 2024-11-21T11:14:59,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:14:59,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42a6ff9198257177a61e80b1f764ee4c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:14:59,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:59,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42a6ff9198257177a61e80b1f764ee4c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:14:59,575 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:14:59,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:14:59,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42a6ff9198257177a61e80b1f764ee4c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:14:59,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:14:59,575 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:14:59,576 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:14:59,576 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:14:59,576 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 42a6ff9198257177a61e80b1f764ee4c/A is initiating minor compaction (all files) 2024-11-21T11:14:59,576 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 42a6ff9198257177a61e80b1f764ee4c/B is initiating minor compaction (all files) 2024-11-21T11:14:59,576 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42a6ff9198257177a61e80b1f764ee4c/A in TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:59,576 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42a6ff9198257177a61e80b1f764ee4c/B in TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:59,576 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/c3b1ea763acf4d68bdb8fbeea0435f7c, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/8a0d11d9c5c8450f9cc0799d1509123e, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/b86ab614a948444ebcf0726d460aecec] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp, totalSize=33.5 K 2024-11-21T11:14:59,576 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/d56be3c28620423f8009068c723242d0, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/348899dbf6aa4498a8bd15b0cebefc9b, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/76a6b6185488406aa8484039b1173020] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp, totalSize=33.5 K 2024-11-21T11:14:59,577 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting d56be3c28620423f8009068c723242d0, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732187695868 2024-11-21T11:14:59,577 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3b1ea763acf4d68bdb8fbeea0435f7c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732187695868 2024-11-21T11:14:59,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-21T11:14:59,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:59,577 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 42a6ff9198257177a61e80b1f764ee4c 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-21T11:14:59,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=A 2024-11-21T11:14:59,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:14:59,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=B 2024-11-21T11:14:59,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:14:59,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=C 2024-11-21T11:14:59,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:14:59,580 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 348899dbf6aa4498a8bd15b0cebefc9b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732187696519 2024-11-21T11:14:59,580 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a0d11d9c5c8450f9cc0799d1509123e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732187696519 2024-11-21T11:14:59,580 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting b86ab614a948444ebcf0726d460aecec, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732187698700 2024-11-21T11:14:59,580 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 76a6b6185488406aa8484039b1173020, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732187698700 2024-11-21T11:14:59,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/01515dffc14b4a3bbc23f78262f03418 is 50, key is test_row_0/A:col10/1732187698783/Put/seqid=0 2024-11-21T11:14:59,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742122_1298 (size=12151) 2024-11-21T11:14:59,594 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42a6ff9198257177a61e80b1f764ee4c#B#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:59,595 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/279bafc49d76413fa577b8910b1660eb is 50, key is test_row_0/B:col10/1732187696532/Put/seqid=0 2024-11-21T11:14:59,596 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/01515dffc14b4a3bbc23f78262f03418 2024-11-21T11:14:59,598 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42a6ff9198257177a61e80b1f764ee4c#A#compaction#247 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:59,598 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/df6828e60d404c7c943e617c1b811787 is 50, key is test_row_0/A:col10/1732187696532/Put/seqid=0 2024-11-21T11:14:59,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742123_1299 (size=12493) 2024-11-21T11:14:59,612 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/279bafc49d76413fa577b8910b1660eb as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/279bafc49d76413fa577b8910b1660eb 2024-11-21T11:14:59,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742124_1300 (size=12493) 2024-11-21T11:14:59,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/7dbea6e119204279b177a66a0e989879 is 50, key is test_row_0/B:col10/1732187698783/Put/seqid=0 2024-11-21T11:14:59,621 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42a6ff9198257177a61e80b1f764ee4c/B of 42a6ff9198257177a61e80b1f764ee4c into 279bafc49d76413fa577b8910b1660eb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:14:59,621 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:14:59,621 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., storeName=42a6ff9198257177a61e80b1f764ee4c/B, priority=13, startTime=1732187699575; duration=0sec 2024-11-21T11:14:59,621 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:14:59,621 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42a6ff9198257177a61e80b1f764ee4c:B 2024-11-21T11:14:59,621 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:14:59,623 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:14:59,623 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 42a6ff9198257177a61e80b1f764ee4c/C is initiating minor compaction (all files) 2024-11-21T11:14:59,623 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42a6ff9198257177a61e80b1f764ee4c/C in TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:14:59,623 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/cdf2fbba64be4cbeac57f277db4eb00d, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/f30f9add9e1048f1b49bc448e9cb1d0c, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/44ee115bf1ec4d7e99e1438a02b3a677] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp, totalSize=33.5 K 2024-11-21T11:14:59,624 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting cdf2fbba64be4cbeac57f277db4eb00d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732187695868 2024-11-21T11:14:59,624 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting f30f9add9e1048f1b49bc448e9cb1d0c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732187696519 2024-11-21T11:14:59,624 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 44ee115bf1ec4d7e99e1438a02b3a677, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732187698700 2024-11-21T11:14:59,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742125_1301 (size=12151) 2024-11-21T11:14:59,626 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/7dbea6e119204279b177a66a0e989879 2024-11-21T11:14:59,634 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42a6ff9198257177a61e80b1f764ee4c#C#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:14:59,635 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/d64eb8e2ecf94b3f85cef634386ceff7 is 50, key is test_row_0/C:col10/1732187696532/Put/seqid=0 2024-11-21T11:14:59,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/417513c70da6469ea6b18400f3f5d30a is 50, key is test_row_0/C:col10/1732187698783/Put/seqid=0 2024-11-21T11:14:59,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742127_1303 (size=12151) 2024-11-21T11:14:59,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742126_1302 (size=12493) 2024-11-21T11:14:59,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T11:14:59,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 42a6ff9198257177a61e80b1f764ee4c 2024-11-21T11:14:59,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:14:59,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:59,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187759948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:59,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:59,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187759948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:59,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:59,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:59,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:59,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:59,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:59,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:59,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:59,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51768 deadline: 1732187759985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:59,989 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_0' in 42a6ff9198257177a61e80b1f764ee4c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:14:59,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:59,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:14:59,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:14:59,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:59,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51758 deadline: 1732187759995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:14:59,999 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_0' in 42a6ff9198257177a61e80b1f764ee4c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:14:59,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:00,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:00,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:00,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:00,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51744 deadline: 1732187760005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:00,010 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_0' in 42a6ff9198257177a61e80b1f764ee4c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:15:00,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:00,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:00,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:00,018 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/df6828e60d404c7c943e617c1b811787 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/df6828e60d404c7c943e617c1b811787 2024-11-21T11:15:00,026 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42a6ff9198257177a61e80b1f764ee4c/A of 42a6ff9198257177a61e80b1f764ee4c into df6828e60d404c7c943e617c1b811787(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:00,026 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:15:00,026 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., storeName=42a6ff9198257177a61e80b1f764ee4c/A, priority=13, startTime=1732187699575; duration=0sec 2024-11-21T11:15:00,026 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:00,026 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42a6ff9198257177a61e80b1f764ee4c:A 2024-11-21T11:15:00,050 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/417513c70da6469ea6b18400f3f5d30a 2024-11-21T11:15:00,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/01515dffc14b4a3bbc23f78262f03418 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/01515dffc14b4a3bbc23f78262f03418 2024-11-21T11:15:00,057 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/d64eb8e2ecf94b3f85cef634386ceff7 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/d64eb8e2ecf94b3f85cef634386ceff7 2024-11-21T11:15:00,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:00,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187760055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:00,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:00,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:00,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187760055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:00,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:00,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:00,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:00,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:00,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:00,062 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/01515dffc14b4a3bbc23f78262f03418, entries=150, sequenceid=254, filesize=11.9 K 2024-11-21T11:15:00,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/7dbea6e119204279b177a66a0e989879 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/7dbea6e119204279b177a66a0e989879 2024-11-21T11:15:00,065 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42a6ff9198257177a61e80b1f764ee4c/C of 42a6ff9198257177a61e80b1f764ee4c into d64eb8e2ecf94b3f85cef634386ceff7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:00,065 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:15:00,065 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., storeName=42a6ff9198257177a61e80b1f764ee4c/C, priority=13, startTime=1732187699575; duration=0sec 2024-11-21T11:15:00,065 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:00,065 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42a6ff9198257177a61e80b1f764ee4c:C 2024-11-21T11:15:00,068 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/7dbea6e119204279b177a66a0e989879, entries=150, sequenceid=254, filesize=11.9 K 2024-11-21T11:15:00,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/417513c70da6469ea6b18400f3f5d30a as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/417513c70da6469ea6b18400f3f5d30a 2024-11-21T11:15:00,073 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/417513c70da6469ea6b18400f3f5d30a, entries=150, sequenceid=254, filesize=11.9 K 2024-11-21T11:15:00,074 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 42a6ff9198257177a61e80b1f764ee4c in 497ms, sequenceid=254, compaction requested=false 2024-11-21T11:15:00,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:15:00,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:00,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-21T11:15:00,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-21T11:15:00,078 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-21T11:15:00,078 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4160 sec 2024-11-21T11:15:00,080 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.4220 sec 2024-11-21T11:15:00,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 42a6ff9198257177a61e80b1f764ee4c 2024-11-21T11:15:00,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42a6ff9198257177a61e80b1f764ee4c 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-21T11:15:00,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=A 2024-11-21T11:15:00,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:00,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=B 2024-11-21T11:15:00,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:00,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=C 2024-11-21T11:15:00,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:00,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/ca96e05fb8954e379b23687242b8b72b is 50, key is test_row_0/A:col10/1732187699940/Put/seqid=0 2024-11-21T11:15:00,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742128_1304 (size=12301) 2024-11-21T11:15:00,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/ca96e05fb8954e379b23687242b8b72b 2024-11-21T11:15:00,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/4aa9904951a04bcbbad5a9df528efc47 is 50, key is test_row_0/B:col10/1732187699940/Put/seqid=0 2024-11-21T11:15:00,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742129_1305 (size=12301) 2024-11-21T11:15:00,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:00,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187760314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:00,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:00,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187760324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:00,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:00,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:00,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:00,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:00,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187760435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:00,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:00,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187760435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:00,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:00,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:00,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:00,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:00,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:00,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:00,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:00,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187760646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:00,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:00,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:00,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187760646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:00,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:00,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:00,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:00,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:00,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:00,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/4aa9904951a04bcbbad5a9df528efc47 2024-11-21T11:15:00,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/beaa6b51d2b140978de0c532f98d0ae3 is 50, key is test_row_0/C:col10/1732187699940/Put/seqid=0 2024-11-21T11:15:00,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742130_1306 (size=12301) 2024-11-21T11:15:00,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T11:15:00,805 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:15:00,806 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:15:00,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-21T11:15:00,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T11:15:00,808 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:15:00,808 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:15:00,808 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:15:00,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T11:15:00,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:00,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187760955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:00,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:00,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:00,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:00,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:00,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187760956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:00,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-21T11:15:00,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:00,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:15:00,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:00,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:00,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:00,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:00,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:00,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:00,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:01,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-21T11:15:01,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:01,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:15:01,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:01,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:01,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:01,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:01,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/beaa6b51d2b140978de0c532f98d0ae3 2024-11-21T11:15:01,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T11:15:01,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/ca96e05fb8954e379b23687242b8b72b as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/ca96e05fb8954e379b23687242b8b72b 2024-11-21T11:15:01,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/ca96e05fb8954e379b23687242b8b72b, entries=150, sequenceid=269, filesize=12.0 K 2024-11-21T11:15:01,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/4aa9904951a04bcbbad5a9df528efc47 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/4aa9904951a04bcbbad5a9df528efc47 2024-11-21T11:15:01,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/4aa9904951a04bcbbad5a9df528efc47, entries=150, sequenceid=269, filesize=12.0 K 2024-11-21T11:15:01,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/beaa6b51d2b140978de0c532f98d0ae3 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/beaa6b51d2b140978de0c532f98d0ae3 2024-11-21T11:15:01,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/beaa6b51d2b140978de0c532f98d0ae3, entries=150, sequenceid=269, filesize=12.0 K 2024-11-21T11:15:01,174 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 42a6ff9198257177a61e80b1f764ee4c in 906ms, sequenceid=269, compaction requested=true 2024-11-21T11:15:01,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:15:01,174 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:15:01,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42a6ff9198257177a61e80b1f764ee4c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:15:01,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:01,175 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:15:01,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42a6ff9198257177a61e80b1f764ee4c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:15:01,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:01,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42a6ff9198257177a61e80b1f764ee4c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:15:01,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:15:01,177 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36945 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:15:01,177 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 42a6ff9198257177a61e80b1f764ee4c/A is initiating minor compaction (all files) 2024-11-21T11:15:01,177 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42a6ff9198257177a61e80b1f764ee4c/A in TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:01,177 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/df6828e60d404c7c943e617c1b811787, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/01515dffc14b4a3bbc23f78262f03418, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/ca96e05fb8954e379b23687242b8b72b] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp, totalSize=36.1 K 2024-11-21T11:15:01,178 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting df6828e60d404c7c943e617c1b811787, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732187696532 2024-11-21T11:15:01,178 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36945 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:15:01,178 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 42a6ff9198257177a61e80b1f764ee4c/B is initiating minor compaction (all files) 2024-11-21T11:15:01,178 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42a6ff9198257177a61e80b1f764ee4c/B in TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:01,178 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/279bafc49d76413fa577b8910b1660eb, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/7dbea6e119204279b177a66a0e989879, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/4aa9904951a04bcbbad5a9df528efc47] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp, totalSize=36.1 K 2024-11-21T11:15:01,180 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01515dffc14b4a3bbc23f78262f03418, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732187698766 2024-11-21T11:15:01,180 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 279bafc49d76413fa577b8910b1660eb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732187696532 2024-11-21T11:15:01,180 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca96e05fb8954e379b23687242b8b72b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732187699940 2024-11-21T11:15:01,180 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 7dbea6e119204279b177a66a0e989879, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732187698766 2024-11-21T11:15:01,182 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 4aa9904951a04bcbbad5a9df528efc47, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732187699940 2024-11-21T11:15:01,195 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42a6ff9198257177a61e80b1f764ee4c#B#compaction#254 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:15:01,195 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/8ccf5d5c2ec84bfb8a79e8dd0ef4f4fb is 50, key is test_row_0/B:col10/1732187699940/Put/seqid=0 2024-11-21T11:15:01,207 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42a6ff9198257177a61e80b1f764ee4c#A#compaction#255 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:15:01,207 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/11b97b16a2514540b0c7c037b17b0f38 is 50, key is test_row_0/A:col10/1732187699940/Put/seqid=0 2024-11-21T11:15:01,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742132_1308 (size=12745) 2024-11-21T11:15:01,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742131_1307 (size=12745) 2024-11-21T11:15:01,258 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/11b97b16a2514540b0c7c037b17b0f38 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/11b97b16a2514540b0c7c037b17b0f38 2024-11-21T11:15:01,264 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/8ccf5d5c2ec84bfb8a79e8dd0ef4f4fb as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/8ccf5d5c2ec84bfb8a79e8dd0ef4f4fb 2024-11-21T11:15:01,267 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42a6ff9198257177a61e80b1f764ee4c/A of 42a6ff9198257177a61e80b1f764ee4c into 11b97b16a2514540b0c7c037b17b0f38(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:01,267 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:15:01,267 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., storeName=42a6ff9198257177a61e80b1f764ee4c/A, priority=13, startTime=1732187701174; duration=0sec 2024-11-21T11:15:01,267 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:15:01,267 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42a6ff9198257177a61e80b1f764ee4c:A 2024-11-21T11:15:01,268 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:15:01,270 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36945 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:15:01,270 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 42a6ff9198257177a61e80b1f764ee4c/C is initiating minor compaction (all files) 2024-11-21T11:15:01,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-21T11:15:01,270 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42a6ff9198257177a61e80b1f764ee4c/C in TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:01,270 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/d64eb8e2ecf94b3f85cef634386ceff7, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/417513c70da6469ea6b18400f3f5d30a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/beaa6b51d2b140978de0c532f98d0ae3] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp, totalSize=36.1 K 2024-11-21T11:15:01,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:01,270 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 42a6ff9198257177a61e80b1f764ee4c 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-21T11:15:01,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=A 2024-11-21T11:15:01,270 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting d64eb8e2ecf94b3f85cef634386ceff7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732187696532 2024-11-21T11:15:01,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:01,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=B 2024-11-21T11:15:01,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:01,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=C 2024-11-21T11:15:01,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:01,271 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 417513c70da6469ea6b18400f3f5d30a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732187698766 2024-11-21T11:15:01,271 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting beaa6b51d2b140978de0c532f98d0ae3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732187699940 2024-11-21T11:15:01,272 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42a6ff9198257177a61e80b1f764ee4c/B of 42a6ff9198257177a61e80b1f764ee4c into 8ccf5d5c2ec84bfb8a79e8dd0ef4f4fb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:01,272 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:15:01,272 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., storeName=42a6ff9198257177a61e80b1f764ee4c/B, priority=13, startTime=1732187701175; duration=0sec 2024-11-21T11:15:01,272 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:01,272 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42a6ff9198257177a61e80b1f764ee4c:B 2024-11-21T11:15:01,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/0669dae110844caea391fb698492ffcc is 50, key is test_row_0/A:col10/1732187700303/Put/seqid=0 2024-11-21T11:15:01,291 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42a6ff9198257177a61e80b1f764ee4c#C#compaction#257 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:15:01,291 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/cf8c99b8b4ef4674a8df868c72d10a0a is 50, key is test_row_0/C:col10/1732187699940/Put/seqid=0 2024-11-21T11:15:01,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742133_1309 (size=12301) 2024-11-21T11:15:01,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742134_1310 (size=12745) 2024-11-21T11:15:01,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T11:15:01,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:15:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 42a6ff9198257177a61e80b1f764ee4c 2024-11-21T11:15:01,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:01,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187761496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:01,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:01,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:01,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:01,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:01,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187761498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:01,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:01,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:01,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:01,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187761605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:01,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:01,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187761606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:01,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:01,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:01,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:01,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:01,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:01,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:01,701 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/0669dae110844caea391fb698492ffcc 2024-11-21T11:15:01,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/0769f07c27ea45e6a274ce6112cbb4a3 is 50, key is test_row_0/B:col10/1732187700303/Put/seqid=0 2024-11-21T11:15:01,718 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/cf8c99b8b4ef4674a8df868c72d10a0a as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/cf8c99b8b4ef4674a8df868c72d10a0a 2024-11-21T11:15:01,728 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42a6ff9198257177a61e80b1f764ee4c/C of 42a6ff9198257177a61e80b1f764ee4c into cf8c99b8b4ef4674a8df868c72d10a0a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:01,729 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:15:01,729 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., storeName=42a6ff9198257177a61e80b1f764ee4c/C, priority=13, startTime=1732187701175; duration=0sec 2024-11-21T11:15:01,729 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:01,729 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42a6ff9198257177a61e80b1f764ee4c:C 2024-11-21T11:15:01,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742135_1311 (size=12301) 2024-11-21T11:15:01,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:01,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187761815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:01,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:01,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:01,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:01,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:01,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187761824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:01,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:01,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:01,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:01,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T11:15:02,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:02,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187762125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:02,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:02,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:02,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:02,132 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/0769f07c27ea45e6a274ce6112cbb4a3 2024-11-21T11:15:02,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:02,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187762136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:02,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:02,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:02,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:02,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/f3f20ea464034c1db8731424347ff310 is 50, key is test_row_0/C:col10/1732187700303/Put/seqid=0 2024-11-21T11:15:02,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742136_1312 (size=12301) 2024-11-21T11:15:02,547 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/f3f20ea464034c1db8731424347ff310 2024-11-21T11:15:02,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/0669dae110844caea391fb698492ffcc as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/0669dae110844caea391fb698492ffcc 2024-11-21T11:15:02,559 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/0669dae110844caea391fb698492ffcc, entries=150, sequenceid=290, filesize=12.0 K 2024-11-21T11:15:02,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/0769f07c27ea45e6a274ce6112cbb4a3 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/0769f07c27ea45e6a274ce6112cbb4a3 2024-11-21T11:15:02,565 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/0769f07c27ea45e6a274ce6112cbb4a3, entries=150, sequenceid=290, filesize=12.0 K 2024-11-21T11:15:02,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/f3f20ea464034c1db8731424347ff310 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/f3f20ea464034c1db8731424347ff310 2024-11-21T11:15:02,570 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/f3f20ea464034c1db8731424347ff310, entries=150, sequenceid=290, filesize=12.0 K 2024-11-21T11:15:02,571 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 42a6ff9198257177a61e80b1f764ee4c in 1301ms, sequenceid=290, compaction requested=false 2024-11-21T11:15:02,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:15:02,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:02,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-21T11:15:02,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-21T11:15:02,575 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-21T11:15:02,575 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7650 sec 2024-11-21T11:15:02,577 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.7700 sec 2024-11-21T11:15:02,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 42a6ff9198257177a61e80b1f764ee4c 2024-11-21T11:15:02,638 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42a6ff9198257177a61e80b1f764ee4c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-21T11:15:02,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=A 2024-11-21T11:15:02,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:02,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=B 2024-11-21T11:15:02,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:02,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=C 2024-11-21T11:15:02,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:02,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/a71c2fd84a484dd5871cc5456f396752 is 50, key is test_row_0/A:col10/1732187701494/Put/seqid=0 2024-11-21T11:15:02,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742137_1313 (size=12301) 2024-11-21T11:15:02,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:02,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187762709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:02,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:02,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187762710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:02,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:02,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:02,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:02,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:02,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:02,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:02,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:02,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187762825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:02,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:02,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:02,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187762825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:02,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:02,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:02,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:02,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:02,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:02,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T11:15:02,965 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:15:02,966 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:15:02,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-21T11:15:02,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T11:15:02,968 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:15:02,968 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:15:02,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:15:03,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:03,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187763036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:03,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:03,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:03,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:03,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:03,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187763036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:03,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:03,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:03,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:03,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/a71c2fd84a484dd5871cc5456f396752 2024-11-21T11:15:03,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/a6ba9c314a1a40378b7d503c572a8a15 is 50, key is test_row_0/B:col10/1732187701494/Put/seqid=0 2024-11-21T11:15:03,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742138_1314 (size=12301) 2024-11-21T11:15:03,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/a6ba9c314a1a40378b7d503c572a8a15 2024-11-21T11:15:03,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/778f2fceb854479e85aed506823f1243 is 50, key is test_row_0/C:col10/1732187701494/Put/seqid=0 2024-11-21T11:15:03,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742139_1315 (size=12301) 2024-11-21T11:15:03,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T11:15:03,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-21T11:15:03,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:03,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:15:03,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:03,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:03,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:03,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:03,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-21T11:15:03,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:03,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:15:03,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:03,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:03,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:03,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T11:15:03,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:03,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187763345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:03,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:03,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187763346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:03,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:03,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:03,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:03,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:03,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:03,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:03,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-21T11:15:03,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:03,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:15:03,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:03,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:03,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:03,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:03,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/778f2fceb854479e85aed506823f1243 2024-11-21T11:15:03,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/a71c2fd84a484dd5871cc5456f396752 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/a71c2fd84a484dd5871cc5456f396752 2024-11-21T11:15:03,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/a71c2fd84a484dd5871cc5456f396752, entries=150, sequenceid=305, filesize=12.0 K 2024-11-21T11:15:03,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/a6ba9c314a1a40378b7d503c572a8a15 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/a6ba9c314a1a40378b7d503c572a8a15 2024-11-21T11:15:03,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/a6ba9c314a1a40378b7d503c572a8a15, entries=150, sequenceid=305, filesize=12.0 K 2024-11-21T11:15:03,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/778f2fceb854479e85aed506823f1243 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/778f2fceb854479e85aed506823f1243 2024-11-21T11:15:03,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/778f2fceb854479e85aed506823f1243, entries=150, sequenceid=305, filesize=12.0 K 2024-11-21T11:15:03,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 42a6ff9198257177a61e80b1f764ee4c in 855ms, sequenceid=305, compaction requested=true 2024-11-21T11:15:03,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:15:03,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42a6ff9198257177a61e80b1f764ee4c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:15:03,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:03,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42a6ff9198257177a61e80b1f764ee4c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:15:03,493 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:15:03,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:03,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42a6ff9198257177a61e80b1f764ee4c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:15:03,493 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:15:03,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:15:03,495 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:15:03,495 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 42a6ff9198257177a61e80b1f764ee4c/B is initiating minor compaction (all files) 2024-11-21T11:15:03,495 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42a6ff9198257177a61e80b1f764ee4c/B in TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:03,495 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/8ccf5d5c2ec84bfb8a79e8dd0ef4f4fb, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/0769f07c27ea45e6a274ce6112cbb4a3, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/a6ba9c314a1a40378b7d503c572a8a15] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp, totalSize=36.5 K 2024-11-21T11:15:03,495 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:15:03,495 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 42a6ff9198257177a61e80b1f764ee4c/A is initiating minor compaction (all files) 2024-11-21T11:15:03,495 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42a6ff9198257177a61e80b1f764ee4c/A in TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:03,495 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/11b97b16a2514540b0c7c037b17b0f38, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/0669dae110844caea391fb698492ffcc, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/a71c2fd84a484dd5871cc5456f396752] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp, totalSize=36.5 K 2024-11-21T11:15:03,495 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ccf5d5c2ec84bfb8a79e8dd0ef4f4fb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732187699940 2024-11-21T11:15:03,496 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11b97b16a2514540b0c7c037b17b0f38, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732187699940 2024-11-21T11:15:03,496 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 0769f07c27ea45e6a274ce6112cbb4a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732187700294 2024-11-21T11:15:03,496 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0669dae110844caea391fb698492ffcc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732187700294 2024-11-21T11:15:03,496 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting a6ba9c314a1a40378b7d503c572a8a15, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732187701494 2024-11-21T11:15:03,496 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting a71c2fd84a484dd5871cc5456f396752, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732187701494 2024-11-21T11:15:03,504 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42a6ff9198257177a61e80b1f764ee4c#B#compaction#263 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:15:03,505 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/e94564a946ae4f1fb6ec39c530ed4b8b is 50, key is test_row_0/B:col10/1732187701494/Put/seqid=0 2024-11-21T11:15:03,505 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42a6ff9198257177a61e80b1f764ee4c#A#compaction#264 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:15:03,506 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/37b86f6192004ee2b9d5549f7832d098 is 50, key is test_row_0/A:col10/1732187701494/Put/seqid=0 2024-11-21T11:15:03,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742140_1316 (size=12847) 2024-11-21T11:15:03,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742141_1317 (size=12847) 2024-11-21T11:15:03,540 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/e94564a946ae4f1fb6ec39c530ed4b8b as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/e94564a946ae4f1fb6ec39c530ed4b8b 2024-11-21T11:15:03,547 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42a6ff9198257177a61e80b1f764ee4c/B of 42a6ff9198257177a61e80b1f764ee4c into e94564a946ae4f1fb6ec39c530ed4b8b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:03,547 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:15:03,547 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., storeName=42a6ff9198257177a61e80b1f764ee4c/B, priority=13, startTime=1732187703493; duration=0sec 2024-11-21T11:15:03,548 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:15:03,548 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42a6ff9198257177a61e80b1f764ee4c:B 2024-11-21T11:15:03,548 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:15:03,549 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:15:03,550 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 42a6ff9198257177a61e80b1f764ee4c/C is initiating minor compaction (all files) 2024-11-21T11:15:03,550 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42a6ff9198257177a61e80b1f764ee4c/C in TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:03,550 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/cf8c99b8b4ef4674a8df868c72d10a0a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/f3f20ea464034c1db8731424347ff310, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/778f2fceb854479e85aed506823f1243] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp, totalSize=36.5 K 2024-11-21T11:15:03,550 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting cf8c99b8b4ef4674a8df868c72d10a0a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732187699940 2024-11-21T11:15:03,552 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting f3f20ea464034c1db8731424347ff310, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732187700294 2024-11-21T11:15:03,552 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/37b86f6192004ee2b9d5549f7832d098 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/37b86f6192004ee2b9d5549f7832d098 2024-11-21T11:15:03,553 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 778f2fceb854479e85aed506823f1243, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732187701494 2024-11-21T11:15:03,563 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42a6ff9198257177a61e80b1f764ee4c/A of 42a6ff9198257177a61e80b1f764ee4c into 37b86f6192004ee2b9d5549f7832d098(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:03,563 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:15:03,563 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., storeName=42a6ff9198257177a61e80b1f764ee4c/A, priority=13, startTime=1732187703493; duration=0sec 2024-11-21T11:15:03,563 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:03,563 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42a6ff9198257177a61e80b1f764ee4c:A 2024-11-21T11:15:03,566 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42a6ff9198257177a61e80b1f764ee4c#C#compaction#265 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:15:03,567 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/5ca4a3ddaeb247a8b8fe38fc4f900d02 is 50, key is test_row_0/C:col10/1732187701494/Put/seqid=0 2024-11-21T11:15:03,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742142_1318 (size=12847) 2024-11-21T11:15:03,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-21T11:15:03,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:03,582 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 42a6ff9198257177a61e80b1f764ee4c 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-21T11:15:03,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=A 2024-11-21T11:15:03,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:03,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=B 2024-11-21T11:15:03,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:03,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=C 2024-11-21T11:15:03,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:03,582 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/5ca4a3ddaeb247a8b8fe38fc4f900d02 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/5ca4a3ddaeb247a8b8fe38fc4f900d02 2024-11-21T11:15:03,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/44e391d2de574a17912e910cd6f8077f is 50, key is test_row_0/A:col10/1732187702707/Put/seqid=0 2024-11-21T11:15:03,592 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42a6ff9198257177a61e80b1f764ee4c/C of 42a6ff9198257177a61e80b1f764ee4c into 5ca4a3ddaeb247a8b8fe38fc4f900d02(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:03,593 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:15:03,593 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., storeName=42a6ff9198257177a61e80b1f764ee4c/C, priority=13, startTime=1732187703493; duration=0sec 2024-11-21T11:15:03,593 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:03,593 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42a6ff9198257177a61e80b1f764ee4c:C 2024-11-21T11:15:03,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T11:15:03,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742143_1319 (size=12301) 2024-11-21T11:15:03,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 42a6ff9198257177a61e80b1f764ee4c 2024-11-21T11:15:03,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:15:03,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:03,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187763873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:03,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:03,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:03,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:03,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:03,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187763877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:03,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:03,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:03,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:03,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:03,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187763985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:03,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:03,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:03,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:03,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187763985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:03,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:03,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:03,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:03,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:03,998 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/44e391d2de574a17912e910cd6f8077f 2024-11-21T11:15:04,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/025a3b858a484ba2a2f922d6f175eebf is 50, key is test_row_0/B:col10/1732187702707/Put/seqid=0 2024-11-21T11:15:04,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742144_1320 (size=12301) 2024-11-21T11:15:04,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T11:15:04,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:04,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187764195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:04,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:04,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187764195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:04,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:04,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:04,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:04,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:04,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:04,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:04,411 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/025a3b858a484ba2a2f922d6f175eebf 2024-11-21T11:15:04,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/63987f5d194e4fd3a29bfa1d6b69608a is 50, key is test_row_0/C:col10/1732187702707/Put/seqid=0 2024-11-21T11:15:04,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742145_1321 (size=12301) 2024-11-21T11:15:04,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:04,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187764505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:04,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:04,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187764506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:04,826 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/63987f5d194e4fd3a29bfa1d6b69608a 2024-11-21T11:15:04,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/44e391d2de574a17912e910cd6f8077f as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/44e391d2de574a17912e910cd6f8077f 2024-11-21T11:15:04,836 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/44e391d2de574a17912e910cd6f8077f, entries=150, sequenceid=327, filesize=12.0 K 2024-11-21T11:15:04,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/025a3b858a484ba2a2f922d6f175eebf as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/025a3b858a484ba2a2f922d6f175eebf 2024-11-21T11:15:04,841 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/025a3b858a484ba2a2f922d6f175eebf, entries=150, sequenceid=327, filesize=12.0 K 2024-11-21T11:15:04,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/63987f5d194e4fd3a29bfa1d6b69608a as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/63987f5d194e4fd3a29bfa1d6b69608a 2024-11-21T11:15:04,847 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/63987f5d194e4fd3a29bfa1d6b69608a, entries=150, sequenceid=327, filesize=12.0 K 2024-11-21T11:15:04,848 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 42a6ff9198257177a61e80b1f764ee4c in 1267ms, sequenceid=327, compaction requested=false 2024-11-21T11:15:04,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:15:04,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:04,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-21T11:15:04,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-21T11:15:04,854 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-21T11:15:04,854 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8830 sec 2024-11-21T11:15:04,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.8900 sec 2024-11-21T11:15:05,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 42a6ff9198257177a61e80b1f764ee4c 2024-11-21T11:15:05,018 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42a6ff9198257177a61e80b1f764ee4c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-21T11:15:05,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=A 2024-11-21T11:15:05,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:05,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=B 2024-11-21T11:15:05,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:05,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=C 2024-11-21T11:15:05,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:05,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/1de10d4d5e984d54a13a009e34d7bd38 is 50, key is test_row_0/A:col10/1732187705017/Put/seqid=0 2024-11-21T11:15:05,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742146_1322 (size=17181) 2024-11-21T11:15:05,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/1de10d4d5e984d54a13a009e34d7bd38 2024-11-21T11:15:05,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/52e4b3f1a1614e2eab39e9975d2926bb is 50, key is test_row_0/B:col10/1732187705017/Put/seqid=0 2024-11-21T11:15:05,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:05,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:05,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187765075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:05,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187765076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:05,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742147_1323 (size=12301) 2024-11-21T11:15:05,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:05,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:05,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:05,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:05,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:05,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:05,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T11:15:05,115 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:15:05,116 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:15:05,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-21T11:15:05,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T11:15:05,118 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:15:05,119 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:15:05,119 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:15:05,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:05,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187765186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:05,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:05,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187765186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:05,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:05,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:05,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:05,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:05,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:05,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:05,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T11:15:05,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-21T11:15:05,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:05,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:15:05,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:05,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:05,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:05,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:05,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:05,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187765395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:05,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:05,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:05,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187765396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:05,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:05,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:05,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:05,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:05,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:05,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-21T11:15:05,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:05,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:15:05,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:05,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:05,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:05,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:05,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T11:15:05,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/52e4b3f1a1614e2eab39e9975d2926bb 2024-11-21T11:15:05,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/e0cce11761b746fda5a2e977a596708f is 50, key is test_row_0/C:col10/1732187705017/Put/seqid=0 2024-11-21T11:15:05,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742148_1324 (size=12301) 2024-11-21T11:15:05,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-21T11:15:05,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:05,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:15:05,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:05,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:05,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:05,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:05,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187765705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:05,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:05,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:05,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:05,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187765705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:05,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:05,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:05,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:05,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-21T11:15:05,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:05,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:15:05,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:05,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:05,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:05,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T11:15:05,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-21T11:15:05,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:05,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:15:05,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:05,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:05,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:05,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:05,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/e0cce11761b746fda5a2e977a596708f 2024-11-21T11:15:05,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/1de10d4d5e984d54a13a009e34d7bd38 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/1de10d4d5e984d54a13a009e34d7bd38 2024-11-21T11:15:05,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/1de10d4d5e984d54a13a009e34d7bd38, entries=250, sequenceid=342, filesize=16.8 K 2024-11-21T11:15:05,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/52e4b3f1a1614e2eab39e9975d2926bb as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/52e4b3f1a1614e2eab39e9975d2926bb 2024-11-21T11:15:05,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/52e4b3f1a1614e2eab39e9975d2926bb, entries=150, sequenceid=342, filesize=12.0 K 2024-11-21T11:15:05,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/e0cce11761b746fda5a2e977a596708f as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/e0cce11761b746fda5a2e977a596708f 2024-11-21T11:15:05,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/e0cce11761b746fda5a2e977a596708f, entries=150, sequenceid=342, filesize=12.0 K 2024-11-21T11:15:05,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 42a6ff9198257177a61e80b1f764ee4c in 897ms, sequenceid=342, compaction requested=true 2024-11-21T11:15:05,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:15:05,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42a6ff9198257177a61e80b1f764ee4c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:15:05,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:05,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42a6ff9198257177a61e80b1f764ee4c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:15:05,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:05,915 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:15:05,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42a6ff9198257177a61e80b1f764ee4c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:15:05,915 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:15:05,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:15:05,917 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:15:05,917 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 42a6ff9198257177a61e80b1f764ee4c/B is initiating minor compaction (all files) 2024-11-21T11:15:05,917 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42a6ff9198257177a61e80b1f764ee4c/B in TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:05,917 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/e94564a946ae4f1fb6ec39c530ed4b8b, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/025a3b858a484ba2a2f922d6f175eebf, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/52e4b3f1a1614e2eab39e9975d2926bb] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp, totalSize=36.6 K 2024-11-21T11:15:05,917 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42329 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:15:05,917 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 42a6ff9198257177a61e80b1f764ee4c/A is initiating minor compaction (all files) 2024-11-21T11:15:05,917 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42a6ff9198257177a61e80b1f764ee4c/A in TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:05,918 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/37b86f6192004ee2b9d5549f7832d098, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/44e391d2de574a17912e910cd6f8077f, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/1de10d4d5e984d54a13a009e34d7bd38] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp, totalSize=41.3 K 2024-11-21T11:15:05,918 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37b86f6192004ee2b9d5549f7832d098, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732187701494 2024-11-21T11:15:05,918 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting e94564a946ae4f1fb6ec39c530ed4b8b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732187701494 2024-11-21T11:15:05,918 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 025a3b858a484ba2a2f922d6f175eebf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732187702678 2024-11-21T11:15:05,918 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44e391d2de574a17912e910cd6f8077f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732187702678 2024-11-21T11:15:05,919 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1de10d4d5e984d54a13a009e34d7bd38, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732187703866 2024-11-21T11:15:05,919 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 52e4b3f1a1614e2eab39e9975d2926bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732187703866 2024-11-21T11:15:05,927 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42a6ff9198257177a61e80b1f764ee4c#B#compaction#272 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:15:05,928 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/302b2d8ad7ae4805a5ce4e26c2e705ae is 50, key is test_row_0/B:col10/1732187705017/Put/seqid=0 2024-11-21T11:15:05,928 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42a6ff9198257177a61e80b1f764ee4c#A#compaction#273 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:15:05,928 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/d69cd239b9fc404f8583ac66399c9845 is 50, key is test_row_0/A:col10/1732187705017/Put/seqid=0 2024-11-21T11:15:05,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742149_1325 (size=12949) 2024-11-21T11:15:05,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742150_1326 (size=12949) 2024-11-21T11:15:05,939 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/302b2d8ad7ae4805a5ce4e26c2e705ae as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/302b2d8ad7ae4805a5ce4e26c2e705ae 2024-11-21T11:15:05,945 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42a6ff9198257177a61e80b1f764ee4c/B of 42a6ff9198257177a61e80b1f764ee4c into 302b2d8ad7ae4805a5ce4e26c2e705ae(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:05,945 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:15:05,945 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., storeName=42a6ff9198257177a61e80b1f764ee4c/B, priority=13, startTime=1732187705915; duration=0sec 2024-11-21T11:15:05,945 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:15:05,945 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42a6ff9198257177a61e80b1f764ee4c:B 2024-11-21T11:15:05,945 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:15:05,946 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:15:05,946 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 42a6ff9198257177a61e80b1f764ee4c/C is initiating minor compaction (all files) 2024-11-21T11:15:05,946 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42a6ff9198257177a61e80b1f764ee4c/C in TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:05,946 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/5ca4a3ddaeb247a8b8fe38fc4f900d02, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/63987f5d194e4fd3a29bfa1d6b69608a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/e0cce11761b746fda5a2e977a596708f] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp, totalSize=36.6 K 2024-11-21T11:15:05,947 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ca4a3ddaeb247a8b8fe38fc4f900d02, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732187701494 2024-11-21T11:15:05,947 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 63987f5d194e4fd3a29bfa1d6b69608a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732187702678 2024-11-21T11:15:05,948 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting e0cce11761b746fda5a2e977a596708f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732187703866 2024-11-21T11:15:05,956 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42a6ff9198257177a61e80b1f764ee4c#C#compaction#274 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:15:05,957 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/7e0bccbf99664e50b44244797a25f268 is 50, key is test_row_0/C:col10/1732187705017/Put/seqid=0 2024-11-21T11:15:05,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742151_1327 (size=12949) 2024-11-21T11:15:06,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-21T11:15:06,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:06,039 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 42a6ff9198257177a61e80b1f764ee4c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T11:15:06,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=A 2024-11-21T11:15:06,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:06,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=B 2024-11-21T11:15:06,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:06,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=C 2024-11-21T11:15:06,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:06,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/9748b089bfe749ba80c96ab93dbd093a is 50, key is test_row_0/A:col10/1732187705072/Put/seqid=0 2024-11-21T11:15:06,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742152_1328 (size=12301) 2024-11-21T11:15:06,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:15:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 42a6ff9198257177a61e80b1f764ee4c 2024-11-21T11:15:06,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187766235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:06,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:06,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:06,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:06,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187766240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:06,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:06,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:06,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T11:15:06,340 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/d69cd239b9fc404f8583ac66399c9845 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/d69cd239b9fc404f8583ac66399c9845 2024-11-21T11:15:06,345 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42a6ff9198257177a61e80b1f764ee4c/A of 42a6ff9198257177a61e80b1f764ee4c into d69cd239b9fc404f8583ac66399c9845(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:06,346 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:15:06,346 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., storeName=42a6ff9198257177a61e80b1f764ee4c/A, priority=13, startTime=1732187705915; duration=0sec 2024-11-21T11:15:06,346 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:06,346 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42a6ff9198257177a61e80b1f764ee4c:A 2024-11-21T11:15:06,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187766345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:06,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:06,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:06,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:06,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187766355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:06,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:06,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:06,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:06,367 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/7e0bccbf99664e50b44244797a25f268 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/7e0bccbf99664e50b44244797a25f268 2024-11-21T11:15:06,373 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42a6ff9198257177a61e80b1f764ee4c/C of 42a6ff9198257177a61e80b1f764ee4c into 7e0bccbf99664e50b44244797a25f268(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:06,373 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:15:06,373 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., storeName=42a6ff9198257177a61e80b1f764ee4c/C, priority=13, startTime=1732187705915; duration=0sec 2024-11-21T11:15:06,373 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:06,373 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42a6ff9198257177a61e80b1f764ee4c:C 2024-11-21T11:15:06,456 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/9748b089bfe749ba80c96ab93dbd093a 2024-11-21T11:15:06,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/f4f1266bb489420187bb58f8abb8edf4 is 50, key is test_row_0/B:col10/1732187705072/Put/seqid=0 2024-11-21T11:15:06,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742153_1329 (size=12301) 2024-11-21T11:15:06,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187766555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:06,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:06,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:06,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:06,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:06,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187766566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:06,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:06,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:06,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:06,656 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T11:15:06,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:06,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187766865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:06,868 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/f4f1266bb489420187bb58f8abb8edf4 2024-11-21T11:15:06,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:06,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187766875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:06,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:06,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:06,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:06,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/943e2bb00f8b45fb86a5ae71456622da is 50, key is test_row_0/C:col10/1732187705072/Put/seqid=0 2024-11-21T11:15:06,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742154_1330 (size=12301) 2024-11-21T11:15:07,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T11:15:07,290 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/943e2bb00f8b45fb86a5ae71456622da 2024-11-21T11:15:07,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/9748b089bfe749ba80c96ab93dbd093a as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/9748b089bfe749ba80c96ab93dbd093a 2024-11-21T11:15:07,302 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/9748b089bfe749ba80c96ab93dbd093a, entries=150, sequenceid=362, filesize=12.0 K 2024-11-21T11:15:07,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/f4f1266bb489420187bb58f8abb8edf4 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/f4f1266bb489420187bb58f8abb8edf4 2024-11-21T11:15:07,307 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/f4f1266bb489420187bb58f8abb8edf4, entries=150, sequenceid=362, filesize=12.0 K 2024-11-21T11:15:07,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/943e2bb00f8b45fb86a5ae71456622da as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/943e2bb00f8b45fb86a5ae71456622da 2024-11-21T11:15:07,312 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/943e2bb00f8b45fb86a5ae71456622da, entries=150, sequenceid=362, filesize=12.0 K 2024-11-21T11:15:07,313 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 42a6ff9198257177a61e80b1f764ee4c in 1273ms, sequenceid=362, compaction requested=false 2024-11-21T11:15:07,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:15:07,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:07,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-21T11:15:07,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-21T11:15:07,316 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-21T11:15:07,317 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1960 sec 2024-11-21T11:15:07,318 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 2.2010 sec 2024-11-21T11:15:07,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 42a6ff9198257177a61e80b1f764ee4c 2024-11-21T11:15:07,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42a6ff9198257177a61e80b1f764ee4c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T11:15:07,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=A 2024-11-21T11:15:07,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:07,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=B 2024-11-21T11:15:07,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:07,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=C 2024-11-21T11:15:07,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:07,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/4965fec9a1d347f892a3d91b4e4fc83b is 50, key is test_row_0/A:col10/1732187707377/Put/seqid=0 2024-11-21T11:15:07,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742155_1331 (size=14741) 2024-11-21T11:15:07,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/4965fec9a1d347f892a3d91b4e4fc83b 2024-11-21T11:15:07,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/7801288eeb2745379ae952db750e6e69 is 50, key is test_row_0/B:col10/1732187707377/Put/seqid=0 2024-11-21T11:15:07,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742156_1332 (size=12301) 2024-11-21T11:15:07,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/7801288eeb2745379ae952db750e6e69 2024-11-21T11:15:07,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/dcaf2de7fde940148a5b77644f58017b is 50, key is test_row_0/C:col10/1732187707377/Put/seqid=0 2024-11-21T11:15:07,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742157_1333 (size=12301) 2024-11-21T11:15:07,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:07,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187767424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:07,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:07,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187767425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:07,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:07,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:07,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:07,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:07,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:07,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:07,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:07,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187767536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:07,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:07,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:07,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:07,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:07,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187767536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:07,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:07,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:07,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:07,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:07,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187767745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:07,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:07,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:07,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:07,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:07,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187767745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:07,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:07,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:07,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:07,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/dcaf2de7fde940148a5b77644f58017b 2024-11-21T11:15:07,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/4965fec9a1d347f892a3d91b4e4fc83b as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/4965fec9a1d347f892a3d91b4e4fc83b 2024-11-21T11:15:07,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/4965fec9a1d347f892a3d91b4e4fc83b, entries=200, sequenceid=378, filesize=14.4 K 2024-11-21T11:15:07,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/7801288eeb2745379ae952db750e6e69 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/7801288eeb2745379ae952db750e6e69 2024-11-21T11:15:07,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/7801288eeb2745379ae952db750e6e69, entries=150, sequenceid=378, filesize=12.0 K 2024-11-21T11:15:07,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/dcaf2de7fde940148a5b77644f58017b as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/dcaf2de7fde940148a5b77644f58017b 2024-11-21T11:15:07,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/dcaf2de7fde940148a5b77644f58017b, entries=150, sequenceid=378, filesize=12.0 K 2024-11-21T11:15:07,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 42a6ff9198257177a61e80b1f764ee4c in 454ms, sequenceid=378, compaction requested=true 2024-11-21T11:15:07,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:15:07,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42a6ff9198257177a61e80b1f764ee4c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:15:07,833 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:15:07,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:07,833 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:15:07,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42a6ff9198257177a61e80b1f764ee4c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:15:07,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:07,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42a6ff9198257177a61e80b1f764ee4c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:15:07,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:15:07,834 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:15:07,834 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:15:07,834 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 42a6ff9198257177a61e80b1f764ee4c/A is initiating minor compaction (all files) 2024-11-21T11:15:07,834 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 42a6ff9198257177a61e80b1f764ee4c/B is initiating minor compaction (all files) 2024-11-21T11:15:07,834 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42a6ff9198257177a61e80b1f764ee4c/A in TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:07,834 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42a6ff9198257177a61e80b1f764ee4c/B in TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:07,834 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/d69cd239b9fc404f8583ac66399c9845, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/9748b089bfe749ba80c96ab93dbd093a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/4965fec9a1d347f892a3d91b4e4fc83b] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp, totalSize=39.1 K 2024-11-21T11:15:07,834 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/302b2d8ad7ae4805a5ce4e26c2e705ae, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/f4f1266bb489420187bb58f8abb8edf4, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/7801288eeb2745379ae952db750e6e69] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp, totalSize=36.7 K 2024-11-21T11:15:07,834 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting d69cd239b9fc404f8583ac66399c9845, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732187703866 2024-11-21T11:15:07,834 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 302b2d8ad7ae4805a5ce4e26c2e705ae, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732187703866 2024-11-21T11:15:07,835 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting f4f1266bb489420187bb58f8abb8edf4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732187705028 2024-11-21T11:15:07,835 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9748b089bfe749ba80c96ab93dbd093a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732187705028 2024-11-21T11:15:07,835 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 7801288eeb2745379ae952db750e6e69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732187706228 2024-11-21T11:15:07,835 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4965fec9a1d347f892a3d91b4e4fc83b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732187706228 2024-11-21T11:15:07,844 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42a6ff9198257177a61e80b1f764ee4c#B#compaction#281 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:15:07,844 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42a6ff9198257177a61e80b1f764ee4c#A#compaction#282 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:15:07,845 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/166f8a6329c74d1ba519b645e1b27c8f is 50, key is test_row_0/B:col10/1732187707377/Put/seqid=0 2024-11-21T11:15:07,845 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/2a183c16966646ca98968918435692d5 is 50, key is test_row_0/A:col10/1732187707377/Put/seqid=0 2024-11-21T11:15:07,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742158_1334 (size=13051) 2024-11-21T11:15:07,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742159_1335 (size=13051) 2024-11-21T11:15:07,863 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/2a183c16966646ca98968918435692d5 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/2a183c16966646ca98968918435692d5 2024-11-21T11:15:07,870 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42a6ff9198257177a61e80b1f764ee4c/A of 42a6ff9198257177a61e80b1f764ee4c into 2a183c16966646ca98968918435692d5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:07,870 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:15:07,870 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., storeName=42a6ff9198257177a61e80b1f764ee4c/A, priority=13, startTime=1732187707832; duration=0sec 2024-11-21T11:15:07,870 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:15:07,870 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42a6ff9198257177a61e80b1f764ee4c:A 2024-11-21T11:15:07,870 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:15:07,871 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:15:07,871 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 42a6ff9198257177a61e80b1f764ee4c/C is initiating minor compaction (all files) 2024-11-21T11:15:07,871 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42a6ff9198257177a61e80b1f764ee4c/C in TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:07,871 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/7e0bccbf99664e50b44244797a25f268, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/943e2bb00f8b45fb86a5ae71456622da, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/dcaf2de7fde940148a5b77644f58017b] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp, totalSize=36.7 K 2024-11-21T11:15:07,872 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e0bccbf99664e50b44244797a25f268, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732187703866 2024-11-21T11:15:07,872 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 943e2bb00f8b45fb86a5ae71456622da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732187705028 2024-11-21T11:15:07,873 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcaf2de7fde940148a5b77644f58017b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732187706228 2024-11-21T11:15:07,880 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42a6ff9198257177a61e80b1f764ee4c#C#compaction#283 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:15:07,881 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/99cbefc695ff4a2c91ea3c6e1a6937f1 is 50, key is test_row_0/C:col10/1732187707377/Put/seqid=0 2024-11-21T11:15:07,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742160_1336 (size=13051) 2024-11-21T11:15:07,892 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/99cbefc695ff4a2c91ea3c6e1a6937f1 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/99cbefc695ff4a2c91ea3c6e1a6937f1 2024-11-21T11:15:07,897 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42a6ff9198257177a61e80b1f764ee4c/C of 42a6ff9198257177a61e80b1f764ee4c into 99cbefc695ff4a2c91ea3c6e1a6937f1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:07,897 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:15:07,897 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., storeName=42a6ff9198257177a61e80b1f764ee4c/C, priority=13, startTime=1732187707833; duration=0sec 2024-11-21T11:15:07,897 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:07,897 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42a6ff9198257177a61e80b1f764ee4c:C 2024-11-21T11:15:08,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 42a6ff9198257177a61e80b1f764ee4c 2024-11-21T11:15:08,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42a6ff9198257177a61e80b1f764ee4c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T11:15:08,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=A 2024-11-21T11:15:08,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:08,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=B 2024-11-21T11:15:08,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:08,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=C 2024-11-21T11:15:08,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:08,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/5ac2bac8b71d43eab42a94696017d9fd is 50, key is test_row_0/A:col10/1732187708058/Put/seqid=0 2024-11-21T11:15:08,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742161_1337 (size=14741) 2024-11-21T11:15:08,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:08,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187768078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:08,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:08,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:08,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:08,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:08,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187768083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:08,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:08,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:08,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:08,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:08,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187768186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:08,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:08,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:08,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:08,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:08,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187768195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:08,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:08,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:08,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:08,263 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/166f8a6329c74d1ba519b645e1b27c8f as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/166f8a6329c74d1ba519b645e1b27c8f 2024-11-21T11:15:08,268 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42a6ff9198257177a61e80b1f764ee4c/B of 42a6ff9198257177a61e80b1f764ee4c into 166f8a6329c74d1ba519b645e1b27c8f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:08,268 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:15:08,268 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., storeName=42a6ff9198257177a61e80b1f764ee4c/B, priority=13, startTime=1732187707833; duration=0sec 2024-11-21T11:15:08,268 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:08,268 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42a6ff9198257177a61e80b1f764ee4c:B 2024-11-21T11:15:08,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:08,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187768395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:08,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:08,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:08,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:08,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:08,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187768406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:08,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:08,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:08,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:08,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/5ac2bac8b71d43eab42a94696017d9fd 2024-11-21T11:15:08,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/0d3b65cc51a4400caa7c025250b6d7e5 is 50, key is test_row_0/B:col10/1732187708058/Put/seqid=0 2024-11-21T11:15:08,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742162_1338 (size=12301) 2024-11-21T11:15:08,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/0d3b65cc51a4400caa7c025250b6d7e5 2024-11-21T11:15:08,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/ffe997c21d724ef1ab24219c3a83ac98 is 50, key is test_row_0/C:col10/1732187708058/Put/seqid=0 2024-11-21T11:15:08,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742163_1339 (size=12301) 2024-11-21T11:15:08,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:08,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187768705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:08,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:08,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187768715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:08,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:08,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:08,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:08,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/ffe997c21d724ef1ab24219c3a83ac98 2024-11-21T11:15:08,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/5ac2bac8b71d43eab42a94696017d9fd as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/5ac2bac8b71d43eab42a94696017d9fd 2024-11-21T11:15:08,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/5ac2bac8b71d43eab42a94696017d9fd, entries=200, sequenceid=400, filesize=14.4 K 2024-11-21T11:15:08,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/0d3b65cc51a4400caa7c025250b6d7e5 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/0d3b65cc51a4400caa7c025250b6d7e5 2024-11-21T11:15:08,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/0d3b65cc51a4400caa7c025250b6d7e5, entries=150, sequenceid=400, filesize=12.0 K 2024-11-21T11:15:08,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/ffe997c21d724ef1ab24219c3a83ac98 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/ffe997c21d724ef1ab24219c3a83ac98 2024-11-21T11:15:08,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/ffe997c21d724ef1ab24219c3a83ac98, entries=150, sequenceid=400, filesize=12.0 K 2024-11-21T11:15:08,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 42a6ff9198257177a61e80b1f764ee4c in 857ms, sequenceid=400, compaction requested=false 2024-11-21T11:15:08,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:15:09,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 42a6ff9198257177a61e80b1f764ee4c 2024-11-21T11:15:09,217 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42a6ff9198257177a61e80b1f764ee4c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-21T11:15:09,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=A 2024-11-21T11:15:09,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:09,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=B 2024-11-21T11:15:09,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:09,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=C 2024-11-21T11:15:09,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:09,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/3ac856fc4ad04238944db13b304fe7fa is 50, key is test_row_0/A:col10/1732187708076/Put/seqid=0 2024-11-21T11:15:09,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742164_1340 (size=14741) 2024-11-21T11:15:09,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/3ac856fc4ad04238944db13b304fe7fa 2024-11-21T11:15:09,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/37a43182cf3e4ab498a42a3426aae8ef is 50, key is test_row_0/B:col10/1732187708076/Put/seqid=0 2024-11-21T11:15:09,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742165_1341 (size=12301) 2024-11-21T11:15:09,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187769278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:09,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187769279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T11:15:09,296 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:15:09,296 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:15:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-21T11:15:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-21T11:15:09,298 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:15:09,299 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:15:09,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:15:09,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187769396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:09,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:09,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187769396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:09,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:09,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:09,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:09,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:09,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-21T11:15:09,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-21T11:15:09,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:09,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:15:09,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:09,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:09,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:09,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:09,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-21T11:15:09,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:09,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:15:09,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:09,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:09,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:09,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:09,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187769605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:09,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:09,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:09,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:09,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187769606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:09,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:09,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:09,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:09,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:09,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-21T11:15:09,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/37a43182cf3e4ab498a42a3426aae8ef 2024-11-21T11:15:09,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/52ac62d779a74a19950deecb19776cfb is 50, key is test_row_0/C:col10/1732187708076/Put/seqid=0 2024-11-21T11:15:09,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742166_1342 (size=12301) 2024-11-21T11:15:09,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-21T11:15:09,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:09,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:15:09,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:09,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:09,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:09,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:09,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-21T11:15:09,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:09,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:15:09,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:09,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:09,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:09,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:09,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:09,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187769915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:09,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:09,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:09,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187769915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:09,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:09,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:09,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:09,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:09,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:09,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-21T11:15:10,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:10,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51758 deadline: 1732187770035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:10,037 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_0' in 42a6ff9198257177a61e80b1f764ee4c of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18258 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:15:10,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:10,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:10,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:10,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/52ac62d779a74a19950deecb19776cfb 2024-11-21T11:15:10,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-21T11:15:10,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:10,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:15:10,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:10,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:10,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:10,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:10,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51768 deadline: 1732187770065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:10,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:10,067 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_0' in 42a6ff9198257177a61e80b1f764ee4c of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18287 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:15:10,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:10,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:10,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:10,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/3ac856fc4ad04238944db13b304fe7fa as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/3ac856fc4ad04238944db13b304fe7fa 2024-11-21T11:15:10,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/3ac856fc4ad04238944db13b304fe7fa, entries=200, sequenceid=414, filesize=14.4 K 2024-11-21T11:15:10,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/37a43182cf3e4ab498a42a3426aae8ef as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/37a43182cf3e4ab498a42a3426aae8ef 2024-11-21T11:15:10,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/37a43182cf3e4ab498a42a3426aae8ef, entries=150, sequenceid=414, filesize=12.0 K 2024-11-21T11:15:10,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/52ac62d779a74a19950deecb19776cfb as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/52ac62d779a74a19950deecb19776cfb 2024-11-21T11:15:10,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:10,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51744 deadline: 1732187770085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:10,088 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_0' in 42a6ff9198257177a61e80b1f764ee4c of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18308 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:15:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:10,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/52ac62d779a74a19950deecb19776cfb, entries=150, sequenceid=414, filesize=12.0 K 2024-11-21T11:15:10,090 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 42a6ff9198257177a61e80b1f764ee4c in 873ms, sequenceid=414, compaction requested=true 2024-11-21T11:15:10,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:15:10,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42a6ff9198257177a61e80b1f764ee4c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:15:10,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:10,091 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:15:10,091 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:15:10,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42a6ff9198257177a61e80b1f764ee4c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:15:10,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:10,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42a6ff9198257177a61e80b1f764ee4c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:15:10,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:15:10,092 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42533 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:15:10,092 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 42a6ff9198257177a61e80b1f764ee4c/A is initiating minor compaction (all files) 2024-11-21T11:15:10,092 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42a6ff9198257177a61e80b1f764ee4c/A in TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:10,092 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/2a183c16966646ca98968918435692d5, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/5ac2bac8b71d43eab42a94696017d9fd, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/3ac856fc4ad04238944db13b304fe7fa] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp, totalSize=41.5 K 2024-11-21T11:15:10,092 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:15:10,092 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 42a6ff9198257177a61e80b1f764ee4c/B is initiating minor compaction (all files) 2024-11-21T11:15:10,092 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42a6ff9198257177a61e80b1f764ee4c/B in TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:10,092 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/166f8a6329c74d1ba519b645e1b27c8f, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/0d3b65cc51a4400caa7c025250b6d7e5, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/37a43182cf3e4ab498a42a3426aae8ef] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp, totalSize=36.8 K 2024-11-21T11:15:10,093 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a183c16966646ca98968918435692d5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732187706228 2024-11-21T11:15:10,093 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 166f8a6329c74d1ba519b645e1b27c8f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732187706228 2024-11-21T11:15:10,093 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d3b65cc51a4400caa7c025250b6d7e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1732187707407 2024-11-21T11:15:10,094 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ac2bac8b71d43eab42a94696017d9fd, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1732187707407 2024-11-21T11:15:10,094 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 37a43182cf3e4ab498a42a3426aae8ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732187708076 2024-11-21T11:15:10,094 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ac856fc4ad04238944db13b304fe7fa, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732187708076 2024-11-21T11:15:10,104 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42a6ff9198257177a61e80b1f764ee4c#B#compaction#290 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:15:10,104 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/9db34db2becd4a5b8afa99a36a311b12 is 50, key is test_row_0/B:col10/1732187708076/Put/seqid=0 2024-11-21T11:15:10,114 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42a6ff9198257177a61e80b1f764ee4c#A#compaction#291 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:15:10,115 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/fe7f73dd0a26429591e9beb667afd10b is 50, key is test_row_0/A:col10/1732187708076/Put/seqid=0 2024-11-21T11:15:10,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742167_1343 (size=13153) 2024-11-21T11:15:10,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742168_1344 (size=13153) 2024-11-21T11:15:10,184 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/fe7f73dd0a26429591e9beb667afd10b as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/fe7f73dd0a26429591e9beb667afd10b 2024-11-21T11:15:10,189 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42a6ff9198257177a61e80b1f764ee4c/A of 42a6ff9198257177a61e80b1f764ee4c into fe7f73dd0a26429591e9beb667afd10b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:10,189 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:15:10,189 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., storeName=42a6ff9198257177a61e80b1f764ee4c/A, priority=13, startTime=1732187710091; duration=0sec 2024-11-21T11:15:10,189 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:15:10,189 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42a6ff9198257177a61e80b1f764ee4c:A 2024-11-21T11:15:10,189 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:15:10,190 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:15:10,190 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 42a6ff9198257177a61e80b1f764ee4c/C is initiating minor compaction (all files) 2024-11-21T11:15:10,190 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42a6ff9198257177a61e80b1f764ee4c/C in TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:10,190 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/99cbefc695ff4a2c91ea3c6e1a6937f1, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/ffe997c21d724ef1ab24219c3a83ac98, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/52ac62d779a74a19950deecb19776cfb] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp, totalSize=36.8 K 2024-11-21T11:15:10,191 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99cbefc695ff4a2c91ea3c6e1a6937f1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732187706228 2024-11-21T11:15:10,191 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffe997c21d724ef1ab24219c3a83ac98, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1732187707407 2024-11-21T11:15:10,191 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52ac62d779a74a19950deecb19776cfb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732187708076 2024-11-21T11:15:10,198 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42a6ff9198257177a61e80b1f764ee4c#C#compaction#292 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:15:10,199 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/4f27cc19800148f49b236c301235923d is 50, key is test_row_0/C:col10/1732187708076/Put/seqid=0 2024-11-21T11:15:10,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742169_1345 (size=13153) 2024-11-21T11:15:10,210 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/4f27cc19800148f49b236c301235923d as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/4f27cc19800148f49b236c301235923d 2024-11-21T11:15:10,216 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42a6ff9198257177a61e80b1f764ee4c/C of 42a6ff9198257177a61e80b1f764ee4c into 4f27cc19800148f49b236c301235923d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:10,216 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:15:10,216 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., storeName=42a6ff9198257177a61e80b1f764ee4c/C, priority=13, startTime=1732187710091; duration=0sec 2024-11-21T11:15:10,216 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:10,216 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42a6ff9198257177a61e80b1f764ee4c:C 2024-11-21T11:15:10,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-21T11:15:10,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:10,219 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 42a6ff9198257177a61e80b1f764ee4c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T11:15:10,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=A 2024-11-21T11:15:10,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:10,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=B 2024-11-21T11:15:10,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:10,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=C 2024-11-21T11:15:10,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:10,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/7ca521e9114544f1b1578cdddb775494 is 50, key is test_row_0/A:col10/1732187709267/Put/seqid=0 2024-11-21T11:15:10,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742170_1346 (size=12301) 2024-11-21T11:15:10,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 42a6ff9198257177a61e80b1f764ee4c 2024-11-21T11:15:10,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. as already flushing 2024-11-21T11:15:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-21T11:15:10,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:10,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187770447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:10,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:10,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187770452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:10,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:10,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:10,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:10,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:10,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187770555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:10,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:10,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:10,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:10,566 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/9db34db2becd4a5b8afa99a36a311b12 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/9db34db2becd4a5b8afa99a36a311b12 2024-11-21T11:15:10,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:10,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187770566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:10,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:10,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:10,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:10,574 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42a6ff9198257177a61e80b1f764ee4c/B of 42a6ff9198257177a61e80b1f764ee4c into 9db34db2becd4a5b8afa99a36a311b12(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:10,574 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:15:10,574 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., storeName=42a6ff9198257177a61e80b1f764ee4c/B, priority=13, startTime=1732187710091; duration=0sec 2024-11-21T11:15:10,574 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:10,574 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42a6ff9198257177a61e80b1f764ee4c:B 2024-11-21T11:15:10,631 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/7ca521e9114544f1b1578cdddb775494 2024-11-21T11:15:10,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/c1143afa6327465d8fd6282fbf466b77 is 50, key is test_row_0/B:col10/1732187709267/Put/seqid=0 2024-11-21T11:15:10,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742171_1347 (size=12301) 2024-11-21T11:15:10,645 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/c1143afa6327465d8fd6282fbf466b77 2024-11-21T11:15:10,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/f23a7016f4db48fe8c5e2020afc75610 is 50, key is test_row_0/C:col10/1732187709267/Put/seqid=0 2024-11-21T11:15:10,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742172_1348 (size=12301) 2024-11-21T11:15:10,657 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/f23a7016f4db48fe8c5e2020afc75610 2024-11-21T11:15:10,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/7ca521e9114544f1b1578cdddb775494 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/7ca521e9114544f1b1578cdddb775494 2024-11-21T11:15:10,667 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/7ca521e9114544f1b1578cdddb775494, entries=150, sequenceid=435, filesize=12.0 K 2024-11-21T11:15:10,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/c1143afa6327465d8fd6282fbf466b77 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/c1143afa6327465d8fd6282fbf466b77 2024-11-21T11:15:10,674 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/c1143afa6327465d8fd6282fbf466b77, entries=150, sequenceid=435, filesize=12.0 K 2024-11-21T11:15:10,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/f23a7016f4db48fe8c5e2020afc75610 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/f23a7016f4db48fe8c5e2020afc75610 2024-11-21T11:15:10,679 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/f23a7016f4db48fe8c5e2020afc75610, entries=150, sequenceid=435, filesize=12.0 K 2024-11-21T11:15:10,680 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 42a6ff9198257177a61e80b1f764ee4c in 461ms, sequenceid=435, compaction requested=false 2024-11-21T11:15:10,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:15:10,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:10,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-21T11:15:10,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-21T11:15:10,684 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-21T11:15:10,684 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3830 sec 2024-11-21T11:15:10,686 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.3890 sec 2024-11-21T11:15:10,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 42a6ff9198257177a61e80b1f764ee4c 2024-11-21T11:15:10,768 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42a6ff9198257177a61e80b1f764ee4c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T11:15:10,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=A 2024-11-21T11:15:10,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:10,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=B 2024-11-21T11:15:10,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:10,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=C 2024-11-21T11:15:10,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:10,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/a5788e47bb5547078908e7bd09df9af9 is 50, key is test_row_0/A:col10/1732187710451/Put/seqid=0 2024-11-21T11:15:10,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742173_1349 (size=12301) 2024-11-21T11:15:10,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:10,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187770813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:10,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:10,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:10,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:10,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:10,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187770815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:10,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:10,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:10,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:10,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:10,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187770926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:10,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:10,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187770926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:10,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:10,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:10,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:10,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:10,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:10,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:11,065 INFO [Thread-1245 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1245. 2024-11-21T11:15:11,066 DEBUG [Thread-1245 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:15:11,066 DEBUG [Thread-1245 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:11,066 DEBUG [Thread-1245 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:11,066 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:15:11,066 INFO [Thread-1241 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1241. 2024-11-21T11:15:11,066 DEBUG [Thread-1241 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:15:11,066 DEBUG [Thread-1241 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:11,066 DEBUG [Thread-1241 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:11,066 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:15:11,067 INFO [Thread-1247 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1247. 2024-11-21T11:15:11,067 DEBUG [Thread-1247 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:15:11,067 DEBUG [Thread-1247 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:11,067 DEBUG [Thread-1247 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:11,067 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:15:11,070 INFO [Thread-1243 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1243. 2024-11-21T11:15:11,070 DEBUG [Thread-1243 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:15:11,070 DEBUG [Thread-1243 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:11,070 INFO [Thread-1239 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1239. 2024-11-21T11:15:11,070 DEBUG [Thread-1239 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:15:11,070 DEBUG [Thread-1239 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:11,070 DEBUG [Thread-1243 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:11,070 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:15:11,071 DEBUG [Thread-1239 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:11,071 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:15:11,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:11,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:11,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187771135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:11,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187771135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:11,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:11,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:11,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:11,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:11,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:11,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:11,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/a5788e47bb5547078908e7bd09df9af9 2024-11-21T11:15:11,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/a1e7cd85be0b474bbd35eafe1f9b30f0 is 50, key is test_row_0/B:col10/1732187710451/Put/seqid=0 2024-11-21T11:15:11,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742174_1350 (size=12301) 2024-11-21T11:15:11,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-21T11:15:11,446 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:15:11,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:11,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:11,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187771446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:11,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187771446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:11,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:11,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:11,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:11,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:11,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:11,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:11,589 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/a1e7cd85be0b474bbd35eafe1f9b30f0 2024-11-21T11:15:11,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/aadf22594b3a417cb2a7e20103452805 is 50, key is test_row_0/C:col10/1732187710451/Put/seqid=0 2024-11-21T11:15:11,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742175_1351 (size=12301) 2024-11-21T11:15:11,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:11,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:11,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1732187771955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:11,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51782 deadline: 1732187771955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:11,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:11,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:11,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:11,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:11,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:11,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:11,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/aadf22594b3a417cb2a7e20103452805 2024-11-21T11:15:12,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/a5788e47bb5547078908e7bd09df9af9 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/a5788e47bb5547078908e7bd09df9af9 2024-11-21T11:15:12,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/a5788e47bb5547078908e7bd09df9af9, entries=150, sequenceid=450, filesize=12.0 K 2024-11-21T11:15:12,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/a1e7cd85be0b474bbd35eafe1f9b30f0 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/a1e7cd85be0b474bbd35eafe1f9b30f0 2024-11-21T11:15:12,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/a1e7cd85be0b474bbd35eafe1f9b30f0, entries=150, sequenceid=450, filesize=12.0 K 2024-11-21T11:15:12,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/aadf22594b3a417cb2a7e20103452805 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/aadf22594b3a417cb2a7e20103452805 2024-11-21T11:15:12,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/aadf22594b3a417cb2a7e20103452805, entries=150, sequenceid=450, filesize=12.0 K 2024-11-21T11:15:12,014 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 42a6ff9198257177a61e80b1f764ee4c in 1246ms, sequenceid=450, compaction requested=true 2024-11-21T11:15:12,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:15:12,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42a6ff9198257177a61e80b1f764ee4c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:15:12,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:12,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42a6ff9198257177a61e80b1f764ee4c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:15:12,014 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:15:12,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:12,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42a6ff9198257177a61e80b1f764ee4c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:15:12,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:15:12,014 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:15:12,015 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:15:12,015 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:15:12,015 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 42a6ff9198257177a61e80b1f764ee4c/A is initiating minor compaction (all files) 2024-11-21T11:15:12,015 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 42a6ff9198257177a61e80b1f764ee4c/B is initiating minor compaction (all files) 2024-11-21T11:15:12,015 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42a6ff9198257177a61e80b1f764ee4c/A in TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:12,015 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42a6ff9198257177a61e80b1f764ee4c/B in TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:12,015 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/fe7f73dd0a26429591e9beb667afd10b, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/7ca521e9114544f1b1578cdddb775494, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/a5788e47bb5547078908e7bd09df9af9] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp, totalSize=36.9 K 2024-11-21T11:15:12,015 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/9db34db2becd4a5b8afa99a36a311b12, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/c1143afa6327465d8fd6282fbf466b77, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/a1e7cd85be0b474bbd35eafe1f9b30f0] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp, totalSize=36.9 K 2024-11-21T11:15:12,015 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe7f73dd0a26429591e9beb667afd10b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732187708076 2024-11-21T11:15:12,015 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 9db34db2becd4a5b8afa99a36a311b12, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732187708076 2024-11-21T11:15:12,015 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ca521e9114544f1b1578cdddb775494, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1732187709256 2024-11-21T11:15:12,016 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting c1143afa6327465d8fd6282fbf466b77, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1732187709256 2024-11-21T11:15:12,016 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5788e47bb5547078908e7bd09df9af9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1732187710445 2024-11-21T11:15:12,016 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting a1e7cd85be0b474bbd35eafe1f9b30f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1732187710445 2024-11-21T11:15:12,022 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42a6ff9198257177a61e80b1f764ee4c#B#compaction#300 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:15:12,022 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42a6ff9198257177a61e80b1f764ee4c#A#compaction#299 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:15:12,022 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/4bb9b6b0be8b46dd9c66f97d497dcf74 is 50, key is test_row_0/B:col10/1732187710451/Put/seqid=0 2024-11-21T11:15:12,022 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/4d4db9718aa74a70966a05bbd88ccac9 is 50, key is test_row_0/A:col10/1732187710451/Put/seqid=0 2024-11-21T11:15:12,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742177_1353 (size=13255) 2024-11-21T11:15:12,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742176_1352 (size=13255) 2024-11-21T11:15:12,430 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/4d4db9718aa74a70966a05bbd88ccac9 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/4d4db9718aa74a70966a05bbd88ccac9 2024-11-21T11:15:12,430 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/4bb9b6b0be8b46dd9c66f97d497dcf74 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/4bb9b6b0be8b46dd9c66f97d497dcf74 2024-11-21T11:15:12,434 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42a6ff9198257177a61e80b1f764ee4c/B of 42a6ff9198257177a61e80b1f764ee4c into 4bb9b6b0be8b46dd9c66f97d497dcf74(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:12,434 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42a6ff9198257177a61e80b1f764ee4c/A of 42a6ff9198257177a61e80b1f764ee4c into 4d4db9718aa74a70966a05bbd88ccac9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:12,434 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:15:12,434 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:15:12,434 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., storeName=42a6ff9198257177a61e80b1f764ee4c/B, priority=13, startTime=1732187712014; duration=0sec 2024-11-21T11:15:12,434 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., storeName=42a6ff9198257177a61e80b1f764ee4c/A, priority=13, startTime=1732187712014; duration=0sec 2024-11-21T11:15:12,435 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:15:12,435 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42a6ff9198257177a61e80b1f764ee4c:B 2024-11-21T11:15:12,435 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:15:12,435 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42a6ff9198257177a61e80b1f764ee4c:A 2024-11-21T11:15:12,435 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:15:12,435 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:15:12,435 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 42a6ff9198257177a61e80b1f764ee4c/C is initiating minor compaction (all files) 2024-11-21T11:15:12,436 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42a6ff9198257177a61e80b1f764ee4c/C in TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:12,436 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/4f27cc19800148f49b236c301235923d, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/f23a7016f4db48fe8c5e2020afc75610, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/aadf22594b3a417cb2a7e20103452805] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp, totalSize=36.9 K 2024-11-21T11:15:12,436 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f27cc19800148f49b236c301235923d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732187708076 2024-11-21T11:15:12,436 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting f23a7016f4db48fe8c5e2020afc75610, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1732187709256 2024-11-21T11:15:12,436 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting aadf22594b3a417cb2a7e20103452805, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1732187710445 2024-11-21T11:15:12,442 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42a6ff9198257177a61e80b1f764ee4c#C#compaction#301 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:15:12,443 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/38a503ddd1f84a0b8bff189c4c84a0ac is 50, key is test_row_0/C:col10/1732187710451/Put/seqid=0 2024-11-21T11:15:12,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742178_1354 (size=13255) 2024-11-21T11:15:12,850 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/38a503ddd1f84a0b8bff189c4c84a0ac as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/38a503ddd1f84a0b8bff189c4c84a0ac 2024-11-21T11:15:12,854 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42a6ff9198257177a61e80b1f764ee4c/C of 42a6ff9198257177a61e80b1f764ee4c into 38a503ddd1f84a0b8bff189c4c84a0ac(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:12,854 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:15:12,854 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., storeName=42a6ff9198257177a61e80b1f764ee4c/C, priority=13, startTime=1732187712014; duration=0sec 2024-11-21T11:15:12,854 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:12,855 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42a6ff9198257177a61e80b1f764ee4c:C 2024-11-21T11:15:12,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 42a6ff9198257177a61e80b1f764ee4c 2024-11-21T11:15:12,967 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42a6ff9198257177a61e80b1f764ee4c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T11:15:12,967 INFO [Thread-1230 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1230. 2024-11-21T11:15:12,967 DEBUG [Thread-1230 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:15:12,967 DEBUG [Thread-1230 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:12,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=A 2024-11-21T11:15:12,967 DEBUG [Thread-1230 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:12,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:12,967 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:15:12,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=B 2024-11-21T11:15:12,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:12,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=C 2024-11-21T11:15:12,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:12,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/5022b8e9050644ae9ed3ecd9c22acd82 is 50, key is test_row_0/A:col10/1732187710814/Put/seqid=0 2024-11-21T11:15:12,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742179_1355 (size=12301) 2024-11-21T11:15:12,977 INFO [Thread-1234 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1234. 2024-11-21T11:15:12,977 DEBUG [Thread-1234 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:15:12,977 DEBUG [Thread-1234 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:12,977 DEBUG [Thread-1234 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:12,978 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:15:13,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/5022b8e9050644ae9ed3ecd9c22acd82 2024-11-21T11:15:13,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/cd8c5f61574b405fb896d062a99e31c3 is 50, key is test_row_0/B:col10/1732187710814/Put/seqid=0 2024-11-21T11:15:13,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742180_1356 (size=12301) 2024-11-21T11:15:13,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/cd8c5f61574b405fb896d062a99e31c3 2024-11-21T11:15:13,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/96aef73a97b44e87813b05e6e2d1c6a1 is 50, key is test_row_0/C:col10/1732187710814/Put/seqid=0 2024-11-21T11:15:13,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742181_1357 (size=12301) 2024-11-21T11:15:14,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/96aef73a97b44e87813b05e6e2d1c6a1 2024-11-21T11:15:14,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/5022b8e9050644ae9ed3ecd9c22acd82 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/5022b8e9050644ae9ed3ecd9c22acd82 2024-11-21T11:15:14,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/5022b8e9050644ae9ed3ecd9c22acd82, entries=150, sequenceid=472, filesize=12.0 K 2024-11-21T11:15:14,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/cd8c5f61574b405fb896d062a99e31c3 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/cd8c5f61574b405fb896d062a99e31c3 2024-11-21T11:15:14,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/cd8c5f61574b405fb896d062a99e31c3, entries=150, sequenceid=472, filesize=12.0 K 2024-11-21T11:15:14,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/96aef73a97b44e87813b05e6e2d1c6a1 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/96aef73a97b44e87813b05e6e2d1c6a1 2024-11-21T11:15:14,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/96aef73a97b44e87813b05e6e2d1c6a1, entries=150, sequenceid=472, filesize=12.0 K 2024-11-21T11:15:14,213 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=6.71 KB/6870 for 42a6ff9198257177a61e80b1f764ee4c in 1246ms, sequenceid=472, compaction requested=false 2024-11-21T11:15:14,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42a6ff9198257177a61e80b1f764ee4c: 2024-11-21T11:15:20,098 INFO [Thread-1228 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1228. 2024-11-21T11:15:20,098 DEBUG [Thread-1228 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:15:20,098 DEBUG [Thread-1228 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:20,098 DEBUG [Thread-1228 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:20,098 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:15:20,158 INFO [Thread-1236 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1236. 2024-11-21T11:15:20,158 DEBUG [Thread-1236 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:15:20,158 DEBUG [Thread-1236 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:20,158 DEBUG [Thread-1236 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:20,158 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:15:20,177 INFO [Thread-1232 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1232. 2024-11-21T11:15:20,177 DEBUG [Thread-1232 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:15:20,177 DEBUG [Thread-1232 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:20,177 DEBUG [Thread-1232 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:20,177 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:15:20,177 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T11:15:20,177 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-21T11:15:20,178 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 165 2024-11-21T11:15:20,178 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 11 2024-11-21T11:15:20,178 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 185 2024-11-21T11:15:20,178 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 11 2024-11-21T11:15:20,178 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T11:15:20,178 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3226 2024-11-21T11:15:20,178 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3248 2024-11-21T11:15:20,178 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3181 2024-11-21T11:15:20,178 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3197 2024-11-21T11:15:20,178 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3263 2024-11-21T11:15:20,178 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T11:15:20,178 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T11:15:20,178 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T11:15:20,178 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:20,178 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:20,178 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:15:20,179 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T11:15:20,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T11:15:20,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-21T11:15:20,182 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732187720182"}]},"ts":"1732187720182"} 2024-11-21T11:15:20,184 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T11:15:20,184 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T11:15:20,184 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T11:15:20,186 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42a6ff9198257177a61e80b1f764ee4c, UNASSIGN}] 2024-11-21T11:15:20,186 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42a6ff9198257177a61e80b1f764ee4c, UNASSIGN 2024-11-21T11:15:20,187 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=42a6ff9198257177a61e80b1f764ee4c, regionState=CLOSING, regionLocation=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:20,188 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=42a6ff9198257177a61e80b1f764ee4c, UNASSIGN because future has completed 2024-11-21T11:15:20,189 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T11:15:20,189 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276}] 2024-11-21T11:15:20,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-21T11:15:20,341 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close 42a6ff9198257177a61e80b1f764ee4c 2024-11-21T11:15:20,341 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T11:15:20,341 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing 42a6ff9198257177a61e80b1f764ee4c, disabling compactions & flushes 2024-11-21T11:15:20,342 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:20,342 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:20,342 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. after waiting 0 ms 2024-11-21T11:15:20,342 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:20,342 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(2902): Flushing 42a6ff9198257177a61e80b1f764ee4c 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-21T11:15:20,342 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=A 2024-11-21T11:15:20,342 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:20,342 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=B 2024-11-21T11:15:20,342 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:20,342 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42a6ff9198257177a61e80b1f764ee4c, store=C 2024-11-21T11:15:20,342 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:20,347 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/425f8dce393f411a9db93eef1cd584ea is 50, key is test_row_0/A:col10/1732187720175/Put/seqid=0 2024-11-21T11:15:20,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742182_1358 (size=9857) 2024-11-21T11:15:20,356 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/425f8dce393f411a9db93eef1cd584ea 2024-11-21T11:15:20,364 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/0f841fa8483d42949afcdf76c3b25dfa is 50, key is test_row_0/B:col10/1732187720175/Put/seqid=0 2024-11-21T11:15:20,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742183_1359 (size=9857) 2024-11-21T11:15:20,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-21T11:15:20,770 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/0f841fa8483d42949afcdf76c3b25dfa 2024-11-21T11:15:20,776 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/f63d22925b6b437ea10b9b793f022ced is 50, key is test_row_0/C:col10/1732187720175/Put/seqid=0 2024-11-21T11:15:20,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742184_1360 (size=9857) 2024-11-21T11:15:20,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-21T11:15:21,180 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/f63d22925b6b437ea10b9b793f022ced 2024-11-21T11:15:21,186 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/A/425f8dce393f411a9db93eef1cd584ea as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/425f8dce393f411a9db93eef1cd584ea 2024-11-21T11:15:21,194 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/425f8dce393f411a9db93eef1cd584ea, entries=100, sequenceid=479, filesize=9.6 K 2024-11-21T11:15:21,195 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/B/0f841fa8483d42949afcdf76c3b25dfa as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/0f841fa8483d42949afcdf76c3b25dfa 2024-11-21T11:15:21,203 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/0f841fa8483d42949afcdf76c3b25dfa, entries=100, sequenceid=479, filesize=9.6 K 2024-11-21T11:15:21,204 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/.tmp/C/f63d22925b6b437ea10b9b793f022ced as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/f63d22925b6b437ea10b9b793f022ced 2024-11-21T11:15:21,208 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/f63d22925b6b437ea10b9b793f022ced, entries=100, sequenceid=479, filesize=9.6 K 2024-11-21T11:15:21,209 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 42a6ff9198257177a61e80b1f764ee4c in 867ms, sequenceid=479, compaction requested=true 2024-11-21T11:15:21,213 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/21bb9a2481e64179bd77c70937f1cfbd, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/18064a02d71b43bf8c00d4c39e142ebd, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/258caf68ffc04d6aa7d11f9166e4c6d2, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/899647521f59479ab443b14174494fa9, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/c45f6a1f57504bd4a20c28127eb3a1ee, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/d6060cdfd2e341bfb0f9d488cd9d41f9, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/15c92aa29d0c47349202b5f9b5f422d4, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/c3b1ea763acf4d68bdb8fbeea0435f7c, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/8a0d11d9c5c8450f9cc0799d1509123e, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/df6828e60d404c7c943e617c1b811787, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/b86ab614a948444ebcf0726d460aecec, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/01515dffc14b4a3bbc23f78262f03418, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/11b97b16a2514540b0c7c037b17b0f38, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/ca96e05fb8954e379b23687242b8b72b, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/0669dae110844caea391fb698492ffcc, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/37b86f6192004ee2b9d5549f7832d098, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/a71c2fd84a484dd5871cc5456f396752, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/44e391d2de574a17912e910cd6f8077f, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/1de10d4d5e984d54a13a009e34d7bd38, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/d69cd239b9fc404f8583ac66399c9845, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/9748b089bfe749ba80c96ab93dbd093a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/4965fec9a1d347f892a3d91b4e4fc83b, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/2a183c16966646ca98968918435692d5, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/5ac2bac8b71d43eab42a94696017d9fd, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/3ac856fc4ad04238944db13b304fe7fa, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/fe7f73dd0a26429591e9beb667afd10b, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/7ca521e9114544f1b1578cdddb775494, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/a5788e47bb5547078908e7bd09df9af9] to archive 2024-11-21T11:15:21,214 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T11:15:21,216 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/21bb9a2481e64179bd77c70937f1cfbd to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/21bb9a2481e64179bd77c70937f1cfbd 2024-11-21T11:15:21,218 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/18064a02d71b43bf8c00d4c39e142ebd to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/18064a02d71b43bf8c00d4c39e142ebd 2024-11-21T11:15:21,219 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/258caf68ffc04d6aa7d11f9166e4c6d2 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/258caf68ffc04d6aa7d11f9166e4c6d2 2024-11-21T11:15:21,220 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/899647521f59479ab443b14174494fa9 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/899647521f59479ab443b14174494fa9 2024-11-21T11:15:21,222 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/c45f6a1f57504bd4a20c28127eb3a1ee to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/c45f6a1f57504bd4a20c28127eb3a1ee 2024-11-21T11:15:21,224 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/d6060cdfd2e341bfb0f9d488cd9d41f9 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/d6060cdfd2e341bfb0f9d488cd9d41f9 2024-11-21T11:15:21,225 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/15c92aa29d0c47349202b5f9b5f422d4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/15c92aa29d0c47349202b5f9b5f422d4 2024-11-21T11:15:21,227 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/c3b1ea763acf4d68bdb8fbeea0435f7c to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/c3b1ea763acf4d68bdb8fbeea0435f7c 2024-11-21T11:15:21,229 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/8a0d11d9c5c8450f9cc0799d1509123e to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/8a0d11d9c5c8450f9cc0799d1509123e 2024-11-21T11:15:21,230 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/df6828e60d404c7c943e617c1b811787 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/df6828e60d404c7c943e617c1b811787 2024-11-21T11:15:21,233 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/b86ab614a948444ebcf0726d460aecec to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/b86ab614a948444ebcf0726d460aecec 2024-11-21T11:15:21,234 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/01515dffc14b4a3bbc23f78262f03418 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/01515dffc14b4a3bbc23f78262f03418 2024-11-21T11:15:21,237 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/11b97b16a2514540b0c7c037b17b0f38 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/11b97b16a2514540b0c7c037b17b0f38 2024-11-21T11:15:21,243 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/ca96e05fb8954e379b23687242b8b72b to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/ca96e05fb8954e379b23687242b8b72b 2024-11-21T11:15:21,245 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/0669dae110844caea391fb698492ffcc to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/0669dae110844caea391fb698492ffcc 2024-11-21T11:15:21,246 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/37b86f6192004ee2b9d5549f7832d098 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/37b86f6192004ee2b9d5549f7832d098 2024-11-21T11:15:21,248 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/a71c2fd84a484dd5871cc5456f396752 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/a71c2fd84a484dd5871cc5456f396752 2024-11-21T11:15:21,249 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/44e391d2de574a17912e910cd6f8077f to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/44e391d2de574a17912e910cd6f8077f 2024-11-21T11:15:21,250 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/1de10d4d5e984d54a13a009e34d7bd38 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/1de10d4d5e984d54a13a009e34d7bd38 2024-11-21T11:15:21,251 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/d69cd239b9fc404f8583ac66399c9845 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/d69cd239b9fc404f8583ac66399c9845 2024-11-21T11:15:21,252 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/9748b089bfe749ba80c96ab93dbd093a to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/9748b089bfe749ba80c96ab93dbd093a 2024-11-21T11:15:21,254 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/4965fec9a1d347f892a3d91b4e4fc83b to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/4965fec9a1d347f892a3d91b4e4fc83b 2024-11-21T11:15:21,255 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/2a183c16966646ca98968918435692d5 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/2a183c16966646ca98968918435692d5 2024-11-21T11:15:21,256 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/5ac2bac8b71d43eab42a94696017d9fd to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/5ac2bac8b71d43eab42a94696017d9fd 2024-11-21T11:15:21,257 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/3ac856fc4ad04238944db13b304fe7fa to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/3ac856fc4ad04238944db13b304fe7fa 2024-11-21T11:15:21,258 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/fe7f73dd0a26429591e9beb667afd10b to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/fe7f73dd0a26429591e9beb667afd10b 2024-11-21T11:15:21,259 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/7ca521e9114544f1b1578cdddb775494 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/7ca521e9114544f1b1578cdddb775494 2024-11-21T11:15:21,260 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/a5788e47bb5547078908e7bd09df9af9 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/a5788e47bb5547078908e7bd09df9af9 2024-11-21T11:15:21,262 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/5e17ecaa63cc4e77b982d314014b7faf, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/8c5e6139b53c42849e5ac1a98ef9aaf2, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/86e1e6dc5bd04c3388414321802bac2f, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/eda8c547ea804e138cf9203e1b722f29, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/391a591facfd43cabeb19347a0a7f1e4, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/e5cba1ebada74f72bf466d13299abfac, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/d56be3c28620423f8009068c723242d0, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/a98fdba762074a669c08d3448ef297f4, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/348899dbf6aa4498a8bd15b0cebefc9b, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/279bafc49d76413fa577b8910b1660eb, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/76a6b6185488406aa8484039b1173020, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/7dbea6e119204279b177a66a0e989879, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/8ccf5d5c2ec84bfb8a79e8dd0ef4f4fb, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/4aa9904951a04bcbbad5a9df528efc47, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/0769f07c27ea45e6a274ce6112cbb4a3, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/e94564a946ae4f1fb6ec39c530ed4b8b, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/a6ba9c314a1a40378b7d503c572a8a15, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/025a3b858a484ba2a2f922d6f175eebf, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/302b2d8ad7ae4805a5ce4e26c2e705ae, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/52e4b3f1a1614e2eab39e9975d2926bb, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/f4f1266bb489420187bb58f8abb8edf4, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/166f8a6329c74d1ba519b645e1b27c8f, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/7801288eeb2745379ae952db750e6e69, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/0d3b65cc51a4400caa7c025250b6d7e5, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/9db34db2becd4a5b8afa99a36a311b12, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/37a43182cf3e4ab498a42a3426aae8ef, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/c1143afa6327465d8fd6282fbf466b77, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/a1e7cd85be0b474bbd35eafe1f9b30f0] to archive 2024-11-21T11:15:21,263 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T11:15:21,264 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/5e17ecaa63cc4e77b982d314014b7faf to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/5e17ecaa63cc4e77b982d314014b7faf 2024-11-21T11:15:21,266 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/8c5e6139b53c42849e5ac1a98ef9aaf2 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/8c5e6139b53c42849e5ac1a98ef9aaf2 2024-11-21T11:15:21,267 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/86e1e6dc5bd04c3388414321802bac2f to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/86e1e6dc5bd04c3388414321802bac2f 2024-11-21T11:15:21,268 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/eda8c547ea804e138cf9203e1b722f29 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/eda8c547ea804e138cf9203e1b722f29 2024-11-21T11:15:21,270 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/391a591facfd43cabeb19347a0a7f1e4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/391a591facfd43cabeb19347a0a7f1e4 2024-11-21T11:15:21,271 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/e5cba1ebada74f72bf466d13299abfac to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/e5cba1ebada74f72bf466d13299abfac 2024-11-21T11:15:21,272 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/d56be3c28620423f8009068c723242d0 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/d56be3c28620423f8009068c723242d0 2024-11-21T11:15:21,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/a98fdba762074a669c08d3448ef297f4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/a98fdba762074a669c08d3448ef297f4 2024-11-21T11:15:21,275 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/348899dbf6aa4498a8bd15b0cebefc9b to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/348899dbf6aa4498a8bd15b0cebefc9b 2024-11-21T11:15:21,305 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/279bafc49d76413fa577b8910b1660eb to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/279bafc49d76413fa577b8910b1660eb 2024-11-21T11:15:21,307 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/76a6b6185488406aa8484039b1173020 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/76a6b6185488406aa8484039b1173020 2024-11-21T11:15:21,313 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/7dbea6e119204279b177a66a0e989879 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/7dbea6e119204279b177a66a0e989879 2024-11-21T11:15:21,315 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/8ccf5d5c2ec84bfb8a79e8dd0ef4f4fb to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/8ccf5d5c2ec84bfb8a79e8dd0ef4f4fb 2024-11-21T11:15:21,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-21T11:15:21,317 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/4aa9904951a04bcbbad5a9df528efc47 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/4aa9904951a04bcbbad5a9df528efc47 2024-11-21T11:15:21,318 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/0769f07c27ea45e6a274ce6112cbb4a3 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/0769f07c27ea45e6a274ce6112cbb4a3 2024-11-21T11:15:21,320 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/e94564a946ae4f1fb6ec39c530ed4b8b to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/e94564a946ae4f1fb6ec39c530ed4b8b 2024-11-21T11:15:21,321 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/a6ba9c314a1a40378b7d503c572a8a15 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/a6ba9c314a1a40378b7d503c572a8a15 2024-11-21T11:15:21,323 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/025a3b858a484ba2a2f922d6f175eebf to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/025a3b858a484ba2a2f922d6f175eebf 2024-11-21T11:15:21,324 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/302b2d8ad7ae4805a5ce4e26c2e705ae to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/302b2d8ad7ae4805a5ce4e26c2e705ae 2024-11-21T11:15:21,325 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/52e4b3f1a1614e2eab39e9975d2926bb to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/52e4b3f1a1614e2eab39e9975d2926bb 2024-11-21T11:15:21,326 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/f4f1266bb489420187bb58f8abb8edf4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/f4f1266bb489420187bb58f8abb8edf4 2024-11-21T11:15:21,328 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/166f8a6329c74d1ba519b645e1b27c8f to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/166f8a6329c74d1ba519b645e1b27c8f 2024-11-21T11:15:21,330 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/7801288eeb2745379ae952db750e6e69 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/7801288eeb2745379ae952db750e6e69 2024-11-21T11:15:21,331 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/0d3b65cc51a4400caa7c025250b6d7e5 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/0d3b65cc51a4400caa7c025250b6d7e5 2024-11-21T11:15:21,333 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/9db34db2becd4a5b8afa99a36a311b12 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/9db34db2becd4a5b8afa99a36a311b12 2024-11-21T11:15:21,334 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/37a43182cf3e4ab498a42a3426aae8ef to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/37a43182cf3e4ab498a42a3426aae8ef 2024-11-21T11:15:21,335 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/c1143afa6327465d8fd6282fbf466b77 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/c1143afa6327465d8fd6282fbf466b77 2024-11-21T11:15:21,337 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/a1e7cd85be0b474bbd35eafe1f9b30f0 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/a1e7cd85be0b474bbd35eafe1f9b30f0 2024-11-21T11:15:21,339 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/d9fddd036ec0486fa4d1db0d0a3d8266, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/5ffa0a59829a44c4ab2d7c257bf8d310, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/91c74615f89b460387683339ec635b47, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/7f823aaf74fa43d6929e71ba3ce27ef5, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/cb30b8a98bd84a3eb34cb9b17608efce, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/bb12bb4728c942fa842b39c6fd31dbbf, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/cdf2fbba64be4cbeac57f277db4eb00d, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/40f84aaca53b48e5a92b0ec0cfdacaaf, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/f30f9add9e1048f1b49bc448e9cb1d0c, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/d64eb8e2ecf94b3f85cef634386ceff7, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/44ee115bf1ec4d7e99e1438a02b3a677, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/417513c70da6469ea6b18400f3f5d30a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/cf8c99b8b4ef4674a8df868c72d10a0a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/beaa6b51d2b140978de0c532f98d0ae3, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/f3f20ea464034c1db8731424347ff310, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/5ca4a3ddaeb247a8b8fe38fc4f900d02, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/778f2fceb854479e85aed506823f1243, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/63987f5d194e4fd3a29bfa1d6b69608a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/7e0bccbf99664e50b44244797a25f268, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/e0cce11761b746fda5a2e977a596708f, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/943e2bb00f8b45fb86a5ae71456622da, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/99cbefc695ff4a2c91ea3c6e1a6937f1, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/dcaf2de7fde940148a5b77644f58017b, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/ffe997c21d724ef1ab24219c3a83ac98, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/4f27cc19800148f49b236c301235923d, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/52ac62d779a74a19950deecb19776cfb, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/f23a7016f4db48fe8c5e2020afc75610, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/aadf22594b3a417cb2a7e20103452805] to archive 2024-11-21T11:15:21,340 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T11:15:21,341 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/d9fddd036ec0486fa4d1db0d0a3d8266 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/d9fddd036ec0486fa4d1db0d0a3d8266 2024-11-21T11:15:21,343 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/5ffa0a59829a44c4ab2d7c257bf8d310 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/5ffa0a59829a44c4ab2d7c257bf8d310 2024-11-21T11:15:21,344 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/91c74615f89b460387683339ec635b47 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/91c74615f89b460387683339ec635b47 2024-11-21T11:15:21,346 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/7f823aaf74fa43d6929e71ba3ce27ef5 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/7f823aaf74fa43d6929e71ba3ce27ef5 2024-11-21T11:15:21,347 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/cb30b8a98bd84a3eb34cb9b17608efce to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/cb30b8a98bd84a3eb34cb9b17608efce 2024-11-21T11:15:21,348 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/bb12bb4728c942fa842b39c6fd31dbbf to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/bb12bb4728c942fa842b39c6fd31dbbf 2024-11-21T11:15:21,350 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/cdf2fbba64be4cbeac57f277db4eb00d to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/cdf2fbba64be4cbeac57f277db4eb00d 2024-11-21T11:15:21,351 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/40f84aaca53b48e5a92b0ec0cfdacaaf to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/40f84aaca53b48e5a92b0ec0cfdacaaf 2024-11-21T11:15:21,352 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/f30f9add9e1048f1b49bc448e9cb1d0c to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/f30f9add9e1048f1b49bc448e9cb1d0c 2024-11-21T11:15:21,354 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/d64eb8e2ecf94b3f85cef634386ceff7 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/d64eb8e2ecf94b3f85cef634386ceff7 2024-11-21T11:15:21,359 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/44ee115bf1ec4d7e99e1438a02b3a677 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/44ee115bf1ec4d7e99e1438a02b3a677 2024-11-21T11:15:21,360 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/417513c70da6469ea6b18400f3f5d30a to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/417513c70da6469ea6b18400f3f5d30a 2024-11-21T11:15:21,362 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/cf8c99b8b4ef4674a8df868c72d10a0a to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/cf8c99b8b4ef4674a8df868c72d10a0a 2024-11-21T11:15:21,363 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/beaa6b51d2b140978de0c532f98d0ae3 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/beaa6b51d2b140978de0c532f98d0ae3 2024-11-21T11:15:21,364 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/f3f20ea464034c1db8731424347ff310 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/f3f20ea464034c1db8731424347ff310 2024-11-21T11:15:21,365 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/5ca4a3ddaeb247a8b8fe38fc4f900d02 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/5ca4a3ddaeb247a8b8fe38fc4f900d02 2024-11-21T11:15:21,366 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/778f2fceb854479e85aed506823f1243 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/778f2fceb854479e85aed506823f1243 2024-11-21T11:15:21,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/63987f5d194e4fd3a29bfa1d6b69608a to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/63987f5d194e4fd3a29bfa1d6b69608a 2024-11-21T11:15:21,368 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/7e0bccbf99664e50b44244797a25f268 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/7e0bccbf99664e50b44244797a25f268 2024-11-21T11:15:21,369 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/e0cce11761b746fda5a2e977a596708f to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/e0cce11761b746fda5a2e977a596708f 2024-11-21T11:15:21,370 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/943e2bb00f8b45fb86a5ae71456622da to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/943e2bb00f8b45fb86a5ae71456622da 2024-11-21T11:15:21,371 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/99cbefc695ff4a2c91ea3c6e1a6937f1 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/99cbefc695ff4a2c91ea3c6e1a6937f1 2024-11-21T11:15:21,372 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/dcaf2de7fde940148a5b77644f58017b to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/dcaf2de7fde940148a5b77644f58017b 2024-11-21T11:15:21,373 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/ffe997c21d724ef1ab24219c3a83ac98 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/ffe997c21d724ef1ab24219c3a83ac98 2024-11-21T11:15:21,374 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/4f27cc19800148f49b236c301235923d to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/4f27cc19800148f49b236c301235923d 2024-11-21T11:15:21,375 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/52ac62d779a74a19950deecb19776cfb to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/52ac62d779a74a19950deecb19776cfb 2024-11-21T11:15:21,376 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/f23a7016f4db48fe8c5e2020afc75610 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/f23a7016f4db48fe8c5e2020afc75610 2024-11-21T11:15:21,377 DEBUG [StoreCloser-TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/aadf22594b3a417cb2a7e20103452805 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/aadf22594b3a417cb2a7e20103452805 2024-11-21T11:15:21,384 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/recovered.edits/482.seqid, newMaxSeqId=482, maxSeqId=1 2024-11-21T11:15:21,384 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. 2024-11-21T11:15:21,385 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for 42a6ff9198257177a61e80b1f764ee4c: Waiting for close lock at 1732187720341Running coprocessor pre-close hooks at 1732187720341Disabling compacts and flushes for region at 1732187720341Disabling writes for close at 1732187720342 (+1 ms)Obtaining lock to block concurrent updates at 1732187720342Preparing flush snapshotting stores in 42a6ff9198257177a61e80b1f764ee4c at 1732187720342Finished memstore snapshotting TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732187720342Flushing stores of TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c. at 1732187720343 (+1 ms)Flushing 42a6ff9198257177a61e80b1f764ee4c/A: creating writer at 1732187720343Flushing 42a6ff9198257177a61e80b1f764ee4c/A: appending metadata at 1732187720347 (+4 ms)Flushing 42a6ff9198257177a61e80b1f764ee4c/A: closing flushed file at 1732187720347Flushing 42a6ff9198257177a61e80b1f764ee4c/B: creating writer at 1732187720360 (+13 ms)Flushing 42a6ff9198257177a61e80b1f764ee4c/B: appending metadata at 1732187720363 (+3 ms)Flushing 42a6ff9198257177a61e80b1f764ee4c/B: closing flushed file at 1732187720363Flushing 42a6ff9198257177a61e80b1f764ee4c/C: creating writer at 1732187720773 (+410 ms)Flushing 42a6ff9198257177a61e80b1f764ee4c/C: appending metadata at 1732187720776 (+3 ms)Flushing 42a6ff9198257177a61e80b1f764ee4c/C: closing flushed file at 1732187720776Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a2a289b: reopening flushed file at 1732187721186 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36ccd929: reopening flushed file at 1732187721194 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@bedda8a: reopening flushed file at 1732187721203 (+9 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 42a6ff9198257177a61e80b1f764ee4c in 867ms, sequenceid=479, compaction requested=true at 1732187721209 (+6 ms)Writing region close event to WAL at 1732187721380 (+171 ms)Running coprocessor post-close hooks at 1732187721384 (+4 ms)Closed at 1732187721384 2024-11-21T11:15:21,386 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed 42a6ff9198257177a61e80b1f764ee4c 2024-11-21T11:15:21,387 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=42a6ff9198257177a61e80b1f764ee4c, regionState=CLOSED 2024-11-21T11:15:21,389 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 because future has completed 2024-11-21T11:15:21,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-21T11:15:21,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure 42a6ff9198257177a61e80b1f764ee4c, server=7b462513bfc2,33417,1732187618276 in 1.2010 sec 2024-11-21T11:15:21,396 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-21T11:15:21,396 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42a6ff9198257177a61e80b1f764ee4c, UNASSIGN in 1.2080 sec 2024-11-21T11:15:21,398 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-21T11:15:21,398 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.2120 sec 2024-11-21T11:15:21,401 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732187721400"}]},"ts":"1732187721400"} 2024-11-21T11:15:21,403 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T11:15:21,403 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T11:15:21,405 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.2250 sec 2024-11-21T11:15:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-21T11:15:22,325 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:15:22,326 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T11:15:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T11:15:22,327 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T11:15:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-21T11:15:22,328 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=90, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T11:15:22,331 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c 2024-11-21T11:15:22,332 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A, FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B, FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C, FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/recovered.edits] 2024-11-21T11:15:22,334 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/425f8dce393f411a9db93eef1cd584ea to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/425f8dce393f411a9db93eef1cd584ea 2024-11-21T11:15:22,335 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/4d4db9718aa74a70966a05bbd88ccac9 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/4d4db9718aa74a70966a05bbd88ccac9 2024-11-21T11:15:22,336 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/5022b8e9050644ae9ed3ecd9c22acd82 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/A/5022b8e9050644ae9ed3ecd9c22acd82 2024-11-21T11:15:22,337 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/0f841fa8483d42949afcdf76c3b25dfa to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/0f841fa8483d42949afcdf76c3b25dfa 2024-11-21T11:15:22,338 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/4bb9b6b0be8b46dd9c66f97d497dcf74 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/4bb9b6b0be8b46dd9c66f97d497dcf74 2024-11-21T11:15:22,339 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/cd8c5f61574b405fb896d062a99e31c3 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/B/cd8c5f61574b405fb896d062a99e31c3 2024-11-21T11:15:22,340 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/38a503ddd1f84a0b8bff189c4c84a0ac to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/38a503ddd1f84a0b8bff189c4c84a0ac 2024-11-21T11:15:22,341 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/96aef73a97b44e87813b05e6e2d1c6a1 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/96aef73a97b44e87813b05e6e2d1c6a1 2024-11-21T11:15:22,342 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/f63d22925b6b437ea10b9b793f022ced to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/C/f63d22925b6b437ea10b9b793f022ced 2024-11-21T11:15:22,344 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/recovered.edits/482.seqid to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c/recovered.edits/482.seqid 2024-11-21T11:15:22,344 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/42a6ff9198257177a61e80b1f764ee4c 2024-11-21T11:15:22,344 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T11:15:22,347 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=90, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T11:15:22,349 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T11:15:22,350 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T11:15:22,351 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=90, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T11:15:22,351 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T11:15:22,352 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732187722351"}]},"ts":"9223372036854775807"} 2024-11-21T11:15:22,353 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T11:15:22,353 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 42a6ff9198257177a61e80b1f764ee4c, NAME => 'TestAcidGuarantees,,1732187689886.42a6ff9198257177a61e80b1f764ee4c.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T11:15:22,353 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T11:15:22,353 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732187722353"}]},"ts":"9223372036854775807"} 2024-11-21T11:15:22,355 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T11:15:22,355 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=90, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T11:15:22,356 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 29 msec 2024-11-21T11:15:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-21T11:15:22,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T11:15:22,435 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:15:22,445 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=246 (was 248), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=440 (was 546), ProcessCount=11 (was 11), AvailableMemoryMB=4689 (was 5401) 2024-11-21T11:15:22,454 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=246, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=440, ProcessCount=11, AvailableMemoryMB=4688 2024-11-21T11:15:22,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T11:15:22,455 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T11:15:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T11:15:22,457 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T11:15:22,457 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:22,457 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 91 2024-11-21T11:15:22,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-21T11:15:22,458 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T11:15:22,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742185_1361 (size=1061) 2024-11-21T11:15:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-21T11:15:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-21T11:15:22,865 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322 2024-11-21T11:15:22,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742186_1362 (size=53) 2024-11-21T11:15:23,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-21T11:15:23,270 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:15:23,270 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d7b388d8fe831b9a9d75fdae52e5e157, disabling compactions & flushes 2024-11-21T11:15:23,270 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:23,270 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:23,270 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. after waiting 0 ms 2024-11-21T11:15:23,270 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:23,270 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:23,270 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d7b388d8fe831b9a9d75fdae52e5e157: Waiting for close lock at 1732187723270Disabling compacts and flushes for region at 1732187723270Disabling writes for close at 1732187723270Writing region close event to WAL at 1732187723270Closed at 1732187723270 2024-11-21T11:15:23,271 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T11:15:23,272 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732187723271"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732187723271"}]},"ts":"1732187723271"} 2024-11-21T11:15:23,273 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T11:15:23,274 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T11:15:23,274 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732187723274"}]},"ts":"1732187723274"} 2024-11-21T11:15:23,275 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T11:15:23,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7b388d8fe831b9a9d75fdae52e5e157, ASSIGN}] 2024-11-21T11:15:23,276 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7b388d8fe831b9a9d75fdae52e5e157, ASSIGN 2024-11-21T11:15:23,277 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7b388d8fe831b9a9d75fdae52e5e157, ASSIGN; state=OFFLINE, location=7b462513bfc2,33417,1732187618276; forceNewPlan=false, retain=false 2024-11-21T11:15:23,428 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=d7b388d8fe831b9a9d75fdae52e5e157, regionState=OPENING, regionLocation=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:23,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7b388d8fe831b9a9d75fdae52e5e157, ASSIGN because future has completed 2024-11-21T11:15:23,430 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276}] 2024-11-21T11:15:23,584 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:23,584 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7752): Opening region: {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} 2024-11-21T11:15:23,585 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:23,585 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:15:23,585 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7794): checking encryption for d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:23,585 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7797): checking classloading for d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:23,586 INFO [StoreOpener-d7b388d8fe831b9a9d75fdae52e5e157-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:23,587 INFO [StoreOpener-d7b388d8fe831b9a9d75fdae52e5e157-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T11:15:23,588 INFO [StoreOpener-d7b388d8fe831b9a9d75fdae52e5e157-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d7b388d8fe831b9a9d75fdae52e5e157 columnFamilyName A 2024-11-21T11:15:23,588 DEBUG [StoreOpener-d7b388d8fe831b9a9d75fdae52e5e157-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:23,588 INFO [StoreOpener-d7b388d8fe831b9a9d75fdae52e5e157-1 {}] regionserver.HStore(327): Store=d7b388d8fe831b9a9d75fdae52e5e157/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:15:23,588 INFO [StoreOpener-d7b388d8fe831b9a9d75fdae52e5e157-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:23,589 INFO [StoreOpener-d7b388d8fe831b9a9d75fdae52e5e157-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T11:15:23,589 INFO [StoreOpener-d7b388d8fe831b9a9d75fdae52e5e157-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d7b388d8fe831b9a9d75fdae52e5e157 columnFamilyName B 2024-11-21T11:15:23,589 DEBUG [StoreOpener-d7b388d8fe831b9a9d75fdae52e5e157-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:23,590 INFO [StoreOpener-d7b388d8fe831b9a9d75fdae52e5e157-1 {}] regionserver.HStore(327): Store=d7b388d8fe831b9a9d75fdae52e5e157/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:15:23,590 INFO [StoreOpener-d7b388d8fe831b9a9d75fdae52e5e157-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:23,590 INFO [StoreOpener-d7b388d8fe831b9a9d75fdae52e5e157-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T11:15:23,591 INFO [StoreOpener-d7b388d8fe831b9a9d75fdae52e5e157-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d7b388d8fe831b9a9d75fdae52e5e157 columnFamilyName C 2024-11-21T11:15:23,591 DEBUG [StoreOpener-d7b388d8fe831b9a9d75fdae52e5e157-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:23,591 INFO [StoreOpener-d7b388d8fe831b9a9d75fdae52e5e157-1 {}] regionserver.HStore(327): Store=d7b388d8fe831b9a9d75fdae52e5e157/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:15:23,591 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:23,591 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1038): replaying wal for d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:23,592 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:23,592 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:23,593 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1048): stopping wal replay for d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:23,593 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1060): Cleaning up temporary data for d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:23,593 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T11:15:23,594 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1093): writing seq id for d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:23,596 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T11:15:23,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-21T11:15:23,596 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1114): Opened d7b388d8fe831b9a9d75fdae52e5e157; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65678020, jitterRate=-0.021321237087249756}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T11:15:23,596 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:23,597 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1006): Region open journal for d7b388d8fe831b9a9d75fdae52e5e157: Running coprocessor pre-open hook at 1732187723585Writing region info on filesystem at 1732187723585Initializing all the Stores at 1732187723586 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187723586Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187723586Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187723586Cleaning up temporary data from old regions at 1732187723593 (+7 ms)Running coprocessor post-open hooks at 1732187723596 (+3 ms)Region opened successfully at 1732187723597 (+1 ms) 2024-11-21T11:15:23,597 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., pid=93, masterSystemTime=1732187723581 2024-11-21T11:15:23,599 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:23,599 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:23,599 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=d7b388d8fe831b9a9d75fdae52e5e157, regionState=OPEN, openSeqNum=2, regionLocation=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:23,601 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 because future has completed 2024-11-21T11:15:23,602 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-21T11:15:23,603 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 in 171 msec 2024-11-21T11:15:23,604 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-21T11:15:23,604 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7b388d8fe831b9a9d75fdae52e5e157, ASSIGN in 328 msec 2024-11-21T11:15:23,605 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T11:15:23,605 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732187723605"}]},"ts":"1732187723605"} 2024-11-21T11:15:23,606 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T11:15:23,607 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T11:15:23,608 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1520 sec 2024-11-21T11:15:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-21T11:15:24,616 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:15:24,617 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@197fefe0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:24,617 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:15:24,617 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:15:24,618 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:15:24,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:15:24,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:15:24,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31fe1017, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:24,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:15:24,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:15:24,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:24,620 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36080, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:15:24,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5500e2df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:24,621 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:15:24,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:15:24,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:15:24,622 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:15:24,623 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T11:15:24,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 7b462513bfc2,36543,1732187617563 2024-11-21T11:15:24,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2a5f5cdd 2024-11-21T11:15:24,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T11:15:24,624 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36090, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T11:15:24,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T11:15:24,626 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T11:15:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-21T11:15:24,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742187_1363 (size=1097) 2024-11-21T11:15:25,036 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-21T11:15:25,037 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-21T11:15:25,039 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T11:15:25,041 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7b388d8fe831b9a9d75fdae52e5e157, REOPEN/MOVE}] 2024-11-21T11:15:25,041 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7b388d8fe831b9a9d75fdae52e5e157, REOPEN/MOVE 2024-11-21T11:15:25,042 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=d7b388d8fe831b9a9d75fdae52e5e157, regionState=CLOSING, regionLocation=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:25,043 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7b388d8fe831b9a9d75fdae52e5e157, REOPEN/MOVE because future has completed 2024-11-21T11:15:25,043 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T11:15:25,043 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276}] 2024-11-21T11:15:25,195 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(122): Close d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:25,196 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T11:15:25,196 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1722): Closing d7b388d8fe831b9a9d75fdae52e5e157, disabling compactions & flushes 2024-11-21T11:15:25,196 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:25,196 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:25,196 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. after waiting 0 ms 2024-11-21T11:15:25,196 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:25,199 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T11:15:25,200 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:25,200 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1676): Region close journal for d7b388d8fe831b9a9d75fdae52e5e157: Waiting for close lock at 1732187725196Running coprocessor pre-close hooks at 1732187725196Disabling compacts and flushes for region at 1732187725196Disabling writes for close at 1732187725196Writing region close event to WAL at 1732187725196Running coprocessor post-close hooks at 1732187725199 (+3 ms)Closed at 1732187725200 (+1 ms) 2024-11-21T11:15:25,200 WARN [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionServer(3299): Not adding moved region record: d7b388d8fe831b9a9d75fdae52e5e157 to self. 2024-11-21T11:15:25,201 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(157): Closed d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:25,202 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=d7b388d8fe831b9a9d75fdae52e5e157, regionState=CLOSED 2024-11-21T11:15:25,203 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 because future has completed 2024-11-21T11:15:25,205 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-21T11:15:25,205 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; CloseRegionProcedure d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 in 160 msec 2024-11-21T11:15:25,205 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7b388d8fe831b9a9d75fdae52e5e157, REOPEN/MOVE; state=CLOSED, location=7b462513bfc2,33417,1732187618276; forceNewPlan=false, retain=true 2024-11-21T11:15:25,356 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=d7b388d8fe831b9a9d75fdae52e5e157, regionState=OPENING, regionLocation=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:25,358 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7b388d8fe831b9a9d75fdae52e5e157, REOPEN/MOVE because future has completed 2024-11-21T11:15:25,358 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276}] 2024-11-21T11:15:25,512 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:25,512 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7752): Opening region: {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} 2024-11-21T11:15:25,513 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:25,513 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:15:25,513 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7794): checking encryption for d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:25,513 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7797): checking classloading for d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:25,514 INFO [StoreOpener-d7b388d8fe831b9a9d75fdae52e5e157-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:25,515 INFO [StoreOpener-d7b388d8fe831b9a9d75fdae52e5e157-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T11:15:25,515 INFO [StoreOpener-d7b388d8fe831b9a9d75fdae52e5e157-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d7b388d8fe831b9a9d75fdae52e5e157 columnFamilyName A 2024-11-21T11:15:25,516 DEBUG [StoreOpener-d7b388d8fe831b9a9d75fdae52e5e157-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:25,517 INFO [StoreOpener-d7b388d8fe831b9a9d75fdae52e5e157-1 {}] regionserver.HStore(327): Store=d7b388d8fe831b9a9d75fdae52e5e157/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:15:25,517 INFO [StoreOpener-d7b388d8fe831b9a9d75fdae52e5e157-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:25,517 INFO [StoreOpener-d7b388d8fe831b9a9d75fdae52e5e157-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T11:15:25,517 INFO [StoreOpener-d7b388d8fe831b9a9d75fdae52e5e157-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d7b388d8fe831b9a9d75fdae52e5e157 columnFamilyName B 2024-11-21T11:15:25,517 DEBUG [StoreOpener-d7b388d8fe831b9a9d75fdae52e5e157-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:25,518 INFO [StoreOpener-d7b388d8fe831b9a9d75fdae52e5e157-1 {}] regionserver.HStore(327): Store=d7b388d8fe831b9a9d75fdae52e5e157/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:15:25,518 INFO [StoreOpener-d7b388d8fe831b9a9d75fdae52e5e157-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:25,518 INFO [StoreOpener-d7b388d8fe831b9a9d75fdae52e5e157-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T11:15:25,519 INFO [StoreOpener-d7b388d8fe831b9a9d75fdae52e5e157-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d7b388d8fe831b9a9d75fdae52e5e157 columnFamilyName C 2024-11-21T11:15:25,519 DEBUG [StoreOpener-d7b388d8fe831b9a9d75fdae52e5e157-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:25,519 INFO [StoreOpener-d7b388d8fe831b9a9d75fdae52e5e157-1 {}] regionserver.HStore(327): Store=d7b388d8fe831b9a9d75fdae52e5e157/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:15:25,519 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:25,519 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1038): replaying wal for d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:25,520 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:25,520 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:25,521 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1048): stopping wal replay for d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:25,521 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1060): Cleaning up temporary data for d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:25,521 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T11:15:25,522 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1093): writing seq id for d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:25,523 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1114): Opened d7b388d8fe831b9a9d75fdae52e5e157; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70054788, jitterRate=0.04389768838882446}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T11:15:25,523 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:25,524 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1006): Region open journal for d7b388d8fe831b9a9d75fdae52e5e157: Running coprocessor pre-open hook at 1732187725513Writing region info on filesystem at 1732187725513Initializing all the Stores at 1732187725514 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187725514Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187725514Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187725514Cleaning up temporary data from old regions at 1732187725521 (+7 ms)Running coprocessor post-open hooks at 1732187725523 (+2 ms)Region opened successfully at 1732187725524 (+1 ms) 2024-11-21T11:15:25,525 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., pid=98, masterSystemTime=1732187725509 2024-11-21T11:15:25,526 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:25,526 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:25,527 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=d7b388d8fe831b9a9d75fdae52e5e157, regionState=OPEN, openSeqNum=5, regionLocation=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:25,528 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 because future has completed 2024-11-21T11:15:25,530 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=96 2024-11-21T11:15:25,530 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=96, state=SUCCESS, hasLock=false; OpenRegionProcedure d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 in 170 msec 2024-11-21T11:15:25,531 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-21T11:15:25,531 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7b388d8fe831b9a9d75fdae52e5e157, REOPEN/MOVE in 489 msec 2024-11-21T11:15:25,533 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-21T11:15:25,533 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 492 msec 2024-11-21T11:15:25,535 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 908 msec 2024-11-21T11:15:25,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-21T11:15:25,536 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:15:25,537 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b2f185f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:25,537 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:15:25,537 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:15:25,538 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:15:25,538 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:15:25,538 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:15:25,538 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1da1c474, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:25,538 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:15:25,538 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:15:25,539 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:25,539 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:15:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e600c98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:25,540 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a739f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:25,540 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:15:25,541 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:15:25,541 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:15:25,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:15:25,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:15:25,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bb7a480, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:25,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:15:25,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:15:25,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:25,542 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36120, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:15:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b4a6f2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:25,543 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@87fd48f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:25,543 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:15:25,543 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:15:25,544 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:15:25,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:15:25,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:15:25,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18014ef5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:25,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:15:25,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:15:25,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:25,545 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:15:25,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cd518eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:25,546 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0bd6a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:25,546 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:15:25,546 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:15:25,547 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:15:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:15:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:15:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb88fae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:15:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:15:25,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:25,548 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:15:25,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47db0f32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:25,549 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b9dbe2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:25,549 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:15:25,549 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:15:25,550 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:15:25,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:15:25,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:15:25,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f21dec1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:25,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:15:25,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:15:25,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:25,551 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36178, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:15:25,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4feb60f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:25,552 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e2b5a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:25,552 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:15:25,552 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:15:25,552 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:15:25,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:15:25,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:15:25,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@402db742, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:25,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:15:25,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:15:25,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:25,553 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:15:25,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31d5bb25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:25,554 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d41a701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:25,554 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:15:25,554 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:15:25,555 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:15:25,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:15:25,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:15:25,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@221c3726, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:25,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:15:25,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:15:25,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:25,556 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:15:25,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c0f9e2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:25,557 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2f46ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:25,557 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:15:25,557 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:15:25,558 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:15:25,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:15:25,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:15:25,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1013d1d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:25,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:15:25,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:15:25,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:25,559 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:15:25,559 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dcfcc17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:25,560 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180bd544, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:25,560 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:15:25,560 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:15:25,560 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:15:25,560 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:15:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:15:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e5bf869, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:15:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:15:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:25,562 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36258, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:15:25,562 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c51e695, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:25,563 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d553196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:25,563 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:15:25,563 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:15:25,563 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:15:25,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:15:25,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:15:25,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a99db35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:25,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:15:25,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:15:25,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:25,565 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36280, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:15:25,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75a11a70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:25,566 DEBUG [Thread-1648 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:15:25,566 DEBUG [Thread-1661 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:15:25,566 DEBUG [Thread-1663 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:15:25,566 DEBUG [Thread-1659 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:15:25,566 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:15:25,566 DEBUG [Thread-1654 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:15:25,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:15:25,566 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:15:25,566 DEBUG [Thread-1665 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:15:25,567 DEBUG [Thread-1646 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:15:25,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:15:25,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:15:25,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:15:25,567 DEBUG [Thread-1657 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:15:25,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-21T11:15:25,567 DEBUG [Thread-1652 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:15:25,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:15:25,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:15:25,568 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:15:25,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:15:25,568 DEBUG [Thread-1650 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:15:25,568 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:15:25,568 DEBUG [Thread-1657 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:15:25,569 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:15:25,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T11:15:25,569 DEBUG [Thread-1657 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:15:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:15:25,569 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35274, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:15:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:15:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:15:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:15:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:15:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:15:25,570 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:15:25,570 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:15:25,570 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35276, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:15:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:15:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:15:25,571 DEBUG [Thread-1650 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:15:25,571 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:15:25,571 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:15:25,571 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:15:25,571 DEBUG [Thread-1650 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:15:25,571 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35312, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:15:25,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5] 2024-11-21T11:15:25,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5] 2024-11-21T11:15:25,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5] 2024-11-21T11:15:25,572 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35324, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:15:25,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5] 2024-11-21T11:15:25,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5] 2024-11-21T11:15:25,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5] 2024-11-21T11:15:25,575 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:15:25,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5] 2024-11-21T11:15:25,575 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5] 2024-11-21T11:15:25,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5] 2024-11-21T11:15:25,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5] 2024-11-21T11:15:25,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:25,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7b388d8fe831b9a9d75fdae52e5e157 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T11:15:25,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=A 2024-11-21T11:15:25,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:25,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=B 2024-11-21T11:15:25,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:25,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=C 2024-11-21T11:15:25,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:25,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112183a5013fdd6c4044819e7518d8b1281d_d7b388d8fe831b9a9d75fdae52e5e157 is 50, key is test_row_0/A:col10/1732187725582/Put/seqid=0 2024-11-21T11:15:25,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742188_1364 (size=9714) 2024-11-21T11:15:25,655 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732187725642, max timestamp=1732187725652Num uniques 859; 2024-11-21T11:15:25,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T11:15:25,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732187725642, max timestamp=1732187725652Num uniques 859; 2024-11-21T11:15:25,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T11:15:25,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732187725642, max timestamp=1732187725652Num uniques 859; 2024-11-21T11:15:25,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T11:15:25,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T11:15:25,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:25,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187785680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:25,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:25,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:25,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:25,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:25,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187785682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:25,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:25,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:25,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35324 deadline: 1732187785683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:25,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:25,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:25,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:25,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:25,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:25,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187785684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:25,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:25,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:25,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187785689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:25,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:25,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:25,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:25,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:25,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:25,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:25,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T11:15:25,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:25,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:25,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:25,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:25,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:25,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:25,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:25,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187785796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:25,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:25,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:25,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:25,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:25,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187785805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:25,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:25,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:25,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:25,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:25,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35324 deadline: 1732187785805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:25,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:25,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:25,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:25,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:25,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187785805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:25,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:25,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:25,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:25,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:25,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187785805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:25,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:25,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:25,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:25,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T11:15:25,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:25,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:25,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:25,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:25,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:25,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:25,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T11:15:26,014 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:26,018 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112183a5013fdd6c4044819e7518d8b1281d_d7b388d8fe831b9a9d75fdae52e5e157 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112183a5013fdd6c4044819e7518d8b1281d_d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:26,019 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/03e9bdbaa36d42a2bc63fbcbde9fb99c, store: [table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:26,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/03e9bdbaa36d42a2bc63fbcbde9fb99c is 175, key is test_row_0/A:col10/1732187725582/Put/seqid=0 2024-11-21T11:15:26,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:26,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187786015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:26,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187786015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:26,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35324 deadline: 1732187786015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:26,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:26,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187786016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:26,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187786016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:26,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T11:15:26,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:26,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:26,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:26,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:26,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:26,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:26,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742189_1365 (size=22361) 2024-11-21T11:15:26,035 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/03e9bdbaa36d42a2bc63fbcbde9fb99c 2024-11-21T11:15:26,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/449de254c9084c4da0d752bca32b6fb1 is 50, key is test_row_0/B:col10/1732187725582/Put/seqid=0 2024-11-21T11:15:26,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742190_1366 (size=9657) 2024-11-21T11:15:26,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/449de254c9084c4da0d752bca32b6fb1 2024-11-21T11:15:26,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/50b7050b35d74bd88ec83ec87baea952 is 50, key is test_row_0/C:col10/1732187725582/Put/seqid=0 2024-11-21T11:15:26,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742191_1367 (size=9657) 2024-11-21T11:15:26,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T11:15:26,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:26,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:26,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:26,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:26,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:26,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:26,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T11:15:26,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187786325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:26,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187786325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:26,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:26,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35324 deadline: 1732187786325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:26,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T11:15:26,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:26,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:26,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:26,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:26,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:26,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187786336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:26,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:26,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187786336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:26,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,342 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:26,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:26,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T11:15:26,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:26,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:26,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:26,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:26,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:26,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:26,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/50b7050b35d74bd88ec83ec87baea952 2024-11-21T11:15:26,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/03e9bdbaa36d42a2bc63fbcbde9fb99c as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/03e9bdbaa36d42a2bc63fbcbde9fb99c 2024-11-21T11:15:26,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/03e9bdbaa36d42a2bc63fbcbde9fb99c, entries=100, sequenceid=14, filesize=21.8 K 2024-11-21T11:15:26,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/449de254c9084c4da0d752bca32b6fb1 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/449de254c9084c4da0d752bca32b6fb1 2024-11-21T11:15:26,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/449de254c9084c4da0d752bca32b6fb1, entries=100, sequenceid=14, filesize=9.4 K 2024-11-21T11:15:26,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/50b7050b35d74bd88ec83ec87baea952 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/50b7050b35d74bd88ec83ec87baea952 2024-11-21T11:15:26,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/50b7050b35d74bd88ec83ec87baea952, entries=100, sequenceid=14, filesize=9.4 K 2024-11-21T11:15:26,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for d7b388d8fe831b9a9d75fdae52e5e157 in 929ms, sequenceid=14, compaction requested=false 2024-11-21T11:15:26,512 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-21T11:15:26,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7b388d8fe831b9a9d75fdae52e5e157: 2024-11-21T11:15:26,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T11:15:26,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:26,643 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing d7b388d8fe831b9a9d75fdae52e5e157 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-21T11:15:26,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=A 2024-11-21T11:15:26,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:15:26,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=B 2024-11-21T11:15:26,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:15:26,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=C 2024-11-21T11:15:26,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:15:26,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219184a8abe1d3470d8ac68f438d7e9f30_d7b388d8fe831b9a9d75fdae52e5e157 is 50, key is test_row_0/A:col10/1732187725664/Put/seqid=0 2024-11-21T11:15:26,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742192_1368 (size=12154) 2024-11-21T11:15:26,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:26,681 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219184a8abe1d3470d8ac68f438d7e9f30_d7b388d8fe831b9a9d75fdae52e5e157 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219184a8abe1d3470d8ac68f438d7e9f30_d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:26,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/702ccec7b0164c19b923d7054f78a588, store: [table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:26,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/702ccec7b0164c19b923d7054f78a588 is 175, key is test_row_0/A:col10/1732187725664/Put/seqid=0 2024-11-21T11:15:26,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742193_1369 (size=30955) 2024-11-21T11:15:26,705 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/702ccec7b0164c19b923d7054f78a588 2024-11-21T11:15:26,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T11:15:26,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/8ee27da56edc4df1a1e6a6cb9f88ff4a is 50, key is test_row_0/B:col10/1732187725664/Put/seqid=0 2024-11-21T11:15:26,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742194_1370 (size=12001) 2024-11-21T11:15:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:26,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:26,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:26,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187786856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:26,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:26,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187786867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:26,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:26,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187786869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:26,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:26,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35324 deadline: 1732187786870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:26,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:26,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187786871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:26,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:26,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:26,905 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T11:15:26,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187786975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:26,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:26,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187786975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:26,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:26,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:26,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187786986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:26,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:26,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35324 deadline: 1732187786986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:26,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:26,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:27,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187786995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:27,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:27,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:27,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:27,122 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/8ee27da56edc4df1a1e6a6cb9f88ff4a 2024-11-21T11:15:27,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/b6409ddd584a42069a51689f961a61f2 is 50, key is test_row_0/C:col10/1732187725664/Put/seqid=0 2024-11-21T11:15:27,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742195_1371 (size=12001) 2024-11-21T11:15:27,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187787186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:27,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:27,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:27,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187787186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:27,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:27,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:27,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:27,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:27,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187787195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:27,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:27,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:27,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:27,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:27,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35324 deadline: 1732187787205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:27,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:27,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:27,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:27,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:27,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187787215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:27,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:27,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187787505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:27,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187787506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:27,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187787506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:27,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:27,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35324 deadline: 1732187787515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:27,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:27,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:27,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:27,538 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/b6409ddd584a42069a51689f961a61f2 2024-11-21T11:15:27,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:27,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187787535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:27,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:27,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:27,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:27,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/702ccec7b0164c19b923d7054f78a588 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/702ccec7b0164c19b923d7054f78a588 2024-11-21T11:15:27,547 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/702ccec7b0164c19b923d7054f78a588, entries=150, sequenceid=50, filesize=30.2 K 2024-11-21T11:15:27,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/8ee27da56edc4df1a1e6a6cb9f88ff4a as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/8ee27da56edc4df1a1e6a6cb9f88ff4a 2024-11-21T11:15:27,551 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/8ee27da56edc4df1a1e6a6cb9f88ff4a, entries=150, sequenceid=50, filesize=11.7 K 2024-11-21T11:15:27,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/b6409ddd584a42069a51689f961a61f2 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/b6409ddd584a42069a51689f961a61f2 2024-11-21T11:15:27,555 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/b6409ddd584a42069a51689f961a61f2, entries=150, sequenceid=50, filesize=11.7 K 2024-11-21T11:15:27,556 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=53.67 KB/54960 for d7b388d8fe831b9a9d75fdae52e5e157 in 913ms, sequenceid=50, compaction requested=false 2024-11-21T11:15:27,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for d7b388d8fe831b9a9d75fdae52e5e157: 2024-11-21T11:15:27,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:27,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-21T11:15:27,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-21T11:15:27,560 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-21T11:15:27,560 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9880 sec 2024-11-21T11:15:27,562 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.9950 sec 2024-11-21T11:15:27,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T11:15:27,726 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:15:27,726 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:15:27,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-21T11:15:27,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T11:15:27,729 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:15:27,729 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:15:27,729 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:15:27,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T11:15:27,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-21T11:15:27,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:27,883 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing d7b388d8fe831b9a9d75fdae52e5e157 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-21T11:15:27,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=A 2024-11-21T11:15:27,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:27,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=B 2024-11-21T11:15:27,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:27,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=C 2024-11-21T11:15:27,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:27,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121126d41074a5f463284ddc456b29b7fb4_d7b388d8fe831b9a9d75fdae52e5e157 is 50, key is test_row_0/A:col10/1732187726868/Put/seqid=0 2024-11-21T11:15:27,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742196_1372 (size=12154) 2024-11-21T11:15:27,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:27,905 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121126d41074a5f463284ddc456b29b7fb4_d7b388d8fe831b9a9d75fdae52e5e157 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121126d41074a5f463284ddc456b29b7fb4_d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:27,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/dd9263d6dc064e66ace9706ff217f9c5, store: [table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:27,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/dd9263d6dc064e66ace9706ff217f9c5 is 175, key is test_row_0/A:col10/1732187726868/Put/seqid=0 2024-11-21T11:15:27,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742197_1373 (size=30955) 2024-11-21T11:15:27,913 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/dd9263d6dc064e66ace9706ff217f9c5 2024-11-21T11:15:27,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/6f8f0d43078644f0b81175f932bd6c24 is 50, key is test_row_0/B:col10/1732187726868/Put/seqid=0 2024-11-21T11:15:27,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742198_1374 (size=12001) 2024-11-21T11:15:27,929 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/6f8f0d43078644f0b81175f932bd6c24 2024-11-21T11:15:27,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/c42292580e0b4db5b37f820741b37e62 is 50, key is test_row_0/C:col10/1732187726868/Put/seqid=0 2024-11-21T11:15:27,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742199_1375 (size=12001) 2024-11-21T11:15:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:28,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:28,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T11:15:28,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732187728058, max timestamp=1732187728074Num uniques 859; 2024-11-21T11:15:28,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732187728058, max timestamp=1732187728075Num uniques 859; 2024-11-21T11:15:28,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T11:15:28,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T11:15:28,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732187728058, max timestamp=1732187728075Num uniques 859; 2024-11-21T11:15:28,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T11:15:28,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:28,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35324 deadline: 1732187788104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:28,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:28,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187788117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:28,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:28,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:28,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:28,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187788118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:28,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:28,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187788120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:28,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187788122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:28,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:28,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35324 deadline: 1732187788225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:28,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:28,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187788225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:28,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:28,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:28,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:28,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187788246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:28,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:28,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187788246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:28,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:28,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:28,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187788246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:28,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:28,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:28,353 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/c42292580e0b4db5b37f820741b37e62 2024-11-21T11:15:28,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T11:15:28,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/dd9263d6dc064e66ace9706ff217f9c5 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/dd9263d6dc064e66ace9706ff217f9c5 2024-11-21T11:15:28,361 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/dd9263d6dc064e66ace9706ff217f9c5, entries=150, sequenceid=61, filesize=30.2 K 2024-11-21T11:15:28,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/6f8f0d43078644f0b81175f932bd6c24 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/6f8f0d43078644f0b81175f932bd6c24 2024-11-21T11:15:28,366 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/6f8f0d43078644f0b81175f932bd6c24, entries=150, sequenceid=61, filesize=11.7 K 2024-11-21T11:15:28,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/c42292580e0b4db5b37f820741b37e62 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/c42292580e0b4db5b37f820741b37e62 2024-11-21T11:15:28,370 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/c42292580e0b4db5b37f820741b37e62, entries=150, sequenceid=61, filesize=11.7 K 2024-11-21T11:15:28,371 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for d7b388d8fe831b9a9d75fdae52e5e157 in 489ms, sequenceid=61, compaction requested=true 2024-11-21T11:15:28,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for d7b388d8fe831b9a9d75fdae52e5e157: 2024-11-21T11:15:28,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:28,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-21T11:15:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-21T11:15:28,375 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-21T11:15:28,375 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 644 msec 2024-11-21T11:15:28,377 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 649 msec 2024-11-21T11:15:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:28,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7b388d8fe831b9a9d75fdae52e5e157 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-21T11:15:28,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=A 2024-11-21T11:15:28,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:15:28,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=B 2024-11-21T11:15:28,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:15:28,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=C 2024-11-21T11:15:28,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:15:28,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212bc92900d35c40e8885c05781789a927_d7b388d8fe831b9a9d75fdae52e5e157 is 50, key is test_row_0/A:col10/1732187728115/Put/seqid=0 2024-11-21T11:15:28,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742200_1376 (size=14594) 2024-11-21T11:15:28,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187788461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:28,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:28,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35324 deadline: 1732187788466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:28,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:28,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187788467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:28,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:28,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187788468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:28,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:28,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187788469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:28,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:28,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:28,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187788575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:28,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:28,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:28,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35324 deadline: 1732187788585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:28,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:28,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187788596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:28,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:28,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187788596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:28,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:28,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187788596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:28,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:28,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:28,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187788795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:28,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:28,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:28,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35324 deadline: 1732187788805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:28,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:28,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:28,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187788816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:28,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:28,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187788816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:28,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:28,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187788816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:28,865 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:28,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T11:15:28,866 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:15:28,866 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:15:28,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-21T11:15:28,868 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:15:28,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T11:15:28,869 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212bc92900d35c40e8885c05781789a927_d7b388d8fe831b9a9d75fdae52e5e157 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212bc92900d35c40e8885c05781789a927_d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:28,869 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:15:28,869 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:15:28,870 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/2e8d1607146a4c959aebf7b00baa76e6, store: [table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:28,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/2e8d1607146a4c959aebf7b00baa76e6 is 175, key is test_row_0/A:col10/1732187728115/Put/seqid=0 2024-11-21T11:15:28,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742201_1377 (size=39549) 2024-11-21T11:15:28,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T11:15:29,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T11:15:29,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:29,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:29,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:29,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:29,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:29,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:29,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187789115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:29,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:29,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:29,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:29,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35324 deadline: 1732187789126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:29,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187789126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:29,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:29,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:29,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:29,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:29,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:29,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187789126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:29,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:29,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:29,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:29,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:29,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187789126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:29,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:29,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:29,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:29,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T11:15:29,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:29,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:29,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:29,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:29,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T11:15:29,275 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/2e8d1607146a4c959aebf7b00baa76e6 2024-11-21T11:15:29,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/f1177a52934c4acc98fbffab3d2ae9c5 is 50, key is test_row_0/B:col10/1732187728115/Put/seqid=0 2024-11-21T11:15:29,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742202_1378 (size=12001) 2024-11-21T11:15:29,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T11:15:29,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:29,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:29,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:29,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:29,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:29,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:29,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T11:15:29,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:29,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:29,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:29,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:29,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:29,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:29,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T11:15:29,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T11:15:29,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:29,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:29,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:29,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:29,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:29,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:29,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:29,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187789635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:29,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:29,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:29,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:29,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:29,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35324 deadline: 1732187789645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:29,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:29,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:29,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187789645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:29,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:29,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:29,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:29,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:29,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:29,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187789646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:29,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:29,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:29,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:29,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187789646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:29,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:29,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:29,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:29,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:29,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:29,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/f1177a52934c4acc98fbffab3d2ae9c5 2024-11-21T11:15:29,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/97e1be61b562412e937da3e24363d4b0 is 50, key is test_row_0/C:col10/1732187728115/Put/seqid=0 2024-11-21T11:15:29,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742203_1379 (size=12001) 2024-11-21T11:15:29,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T11:15:29,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:29,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:29,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:29,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:29,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:29,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:29,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T11:15:29,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:29,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:29,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:29,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:29,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:29,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:30,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T11:15:30,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T11:15:30,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:30,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:30,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:30,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:30,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:30,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:30,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/97e1be61b562412e937da3e24363d4b0 2024-11-21T11:15:30,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/2e8d1607146a4c959aebf7b00baa76e6 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/2e8d1607146a4c959aebf7b00baa76e6 2024-11-21T11:15:30,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/2e8d1607146a4c959aebf7b00baa76e6, entries=200, sequenceid=97, filesize=38.6 K 2024-11-21T11:15:30,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/f1177a52934c4acc98fbffab3d2ae9c5 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/f1177a52934c4acc98fbffab3d2ae9c5 2024-11-21T11:15:30,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/f1177a52934c4acc98fbffab3d2ae9c5, entries=150, sequenceid=97, filesize=11.7 K 2024-11-21T11:15:30,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/97e1be61b562412e937da3e24363d4b0 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/97e1be61b562412e937da3e24363d4b0 2024-11-21T11:15:30,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/97e1be61b562412e937da3e24363d4b0, entries=150, sequenceid=97, filesize=11.7 K 2024-11-21T11:15:30,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for d7b388d8fe831b9a9d75fdae52e5e157 in 1672ms, sequenceid=97, compaction requested=true 2024-11-21T11:15:30,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7b388d8fe831b9a9d75fdae52e5e157: 2024-11-21T11:15:30,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7b388d8fe831b9a9d75fdae52e5e157:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:15:30,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:30,117 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T11:15:30,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7b388d8fe831b9a9d75fdae52e5e157:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:15:30,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:30,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7b388d8fe831b9a9d75fdae52e5e157:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:15:30,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:15:30,117 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T11:15:30,118 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T11:15:30,118 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 123820 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T11:15:30,118 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): d7b388d8fe831b9a9d75fdae52e5e157/B is initiating minor compaction (all files) 2024-11-21T11:15:30,118 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): d7b388d8fe831b9a9d75fdae52e5e157/A is initiating minor compaction (all files) 2024-11-21T11:15:30,118 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7b388d8fe831b9a9d75fdae52e5e157/B in TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:30,118 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7b388d8fe831b9a9d75fdae52e5e157/A in TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:30,118 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/449de254c9084c4da0d752bca32b6fb1, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/8ee27da56edc4df1a1e6a6cb9f88ff4a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/6f8f0d43078644f0b81175f932bd6c24, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/f1177a52934c4acc98fbffab3d2ae9c5] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp, totalSize=44.6 K 2024-11-21T11:15:30,118 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/03e9bdbaa36d42a2bc63fbcbde9fb99c, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/702ccec7b0164c19b923d7054f78a588, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/dd9263d6dc064e66ace9706ff217f9c5, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/2e8d1607146a4c959aebf7b00baa76e6] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp, totalSize=120.9 K 2024-11-21T11:15:30,118 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:30,118 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. files: [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/03e9bdbaa36d42a2bc63fbcbde9fb99c, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/702ccec7b0164c19b923d7054f78a588, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/dd9263d6dc064e66ace9706ff217f9c5, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/2e8d1607146a4c959aebf7b00baa76e6] 2024-11-21T11:15:30,119 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 449de254c9084c4da0d752bca32b6fb1, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732187725580 2024-11-21T11:15:30,119 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03e9bdbaa36d42a2bc63fbcbde9fb99c, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732187725580 2024-11-21T11:15:30,119 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ee27da56edc4df1a1e6a6cb9f88ff4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732187725664 2024-11-21T11:15:30,119 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 702ccec7b0164c19b923d7054f78a588, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732187725664 2024-11-21T11:15:30,119 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f8f0d43078644f0b81175f932bd6c24, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732187726867 2024-11-21T11:15:30,119 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting f1177a52934c4acc98fbffab3d2ae9c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732187728115 2024-11-21T11:15:30,119 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd9263d6dc064e66ace9706ff217f9c5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732187726867 2024-11-21T11:15:30,120 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e8d1607146a4c959aebf7b00baa76e6, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732187728115 2024-11-21T11:15:30,130 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:30,132 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7b388d8fe831b9a9d75fdae52e5e157#B#compaction#320 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:15:30,133 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/60208f1d6cd64cedbbebd4733fae2b21 is 50, key is test_row_0/B:col10/1732187728115/Put/seqid=0 2024-11-21T11:15:30,134 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411217f0f7a444590414aa330b605ff0eeb94_d7b388d8fe831b9a9d75fdae52e5e157 store=[table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:30,137 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411217f0f7a444590414aa330b605ff0eeb94_d7b388d8fe831b9a9d75fdae52e5e157, store=[table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:30,137 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411217f0f7a444590414aa330b605ff0eeb94_d7b388d8fe831b9a9d75fdae52e5e157 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:30,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742204_1380 (size=12139) 2024-11-21T11:15:30,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742205_1381 (size=4469) 2024-11-21T11:15:30,148 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7b388d8fe831b9a9d75fdae52e5e157#A#compaction#321 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:15:30,148 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/4d8665e1cd6f4222ba7eaa291565d67f is 175, key is test_row_0/A:col10/1732187728115/Put/seqid=0 2024-11-21T11:15:30,151 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/60208f1d6cd64cedbbebd4733fae2b21 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/60208f1d6cd64cedbbebd4733fae2b21 2024-11-21T11:15:30,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742206_1382 (size=31093) 2024-11-21T11:15:30,159 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d7b388d8fe831b9a9d75fdae52e5e157/B of d7b388d8fe831b9a9d75fdae52e5e157 into 60208f1d6cd64cedbbebd4733fae2b21(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:30,159 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7b388d8fe831b9a9d75fdae52e5e157: 2024-11-21T11:15:30,159 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., storeName=d7b388d8fe831b9a9d75fdae52e5e157/B, priority=12, startTime=1732187730117; duration=0sec 2024-11-21T11:15:30,159 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:15:30,159 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7b388d8fe831b9a9d75fdae52e5e157:B 2024-11-21T11:15:30,159 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T11:15:30,160 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T11:15:30,160 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): d7b388d8fe831b9a9d75fdae52e5e157/C is initiating minor compaction (all files) 2024-11-21T11:15:30,160 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7b388d8fe831b9a9d75fdae52e5e157/C in TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:30,160 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/50b7050b35d74bd88ec83ec87baea952, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/b6409ddd584a42069a51689f961a61f2, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/c42292580e0b4db5b37f820741b37e62, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/97e1be61b562412e937da3e24363d4b0] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp, totalSize=44.6 K 2024-11-21T11:15:30,163 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 50b7050b35d74bd88ec83ec87baea952, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732187725580 2024-11-21T11:15:30,163 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting b6409ddd584a42069a51689f961a61f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732187725664 2024-11-21T11:15:30,163 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting c42292580e0b4db5b37f820741b37e62, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732187726867 2024-11-21T11:15:30,164 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 97e1be61b562412e937da3e24363d4b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732187728115 2024-11-21T11:15:30,172 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7b388d8fe831b9a9d75fdae52e5e157#C#compaction#322 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:15:30,172 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/071d6a2516be44e08423e49a39ad4972 is 50, key is test_row_0/C:col10/1732187728115/Put/seqid=0 2024-11-21T11:15:30,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742207_1383 (size=12139) 2024-11-21T11:15:30,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T11:15:30,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:30,246 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing d7b388d8fe831b9a9d75fdae52e5e157 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T11:15:30,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=A 2024-11-21T11:15:30,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:30,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=B 2024-11-21T11:15:30,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:30,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=C 2024-11-21T11:15:30,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:30,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e704b0bab36547ec8035ad72e5298fbd_d7b388d8fe831b9a9d75fdae52e5e157 is 50, key is test_row_0/A:col10/1732187728467/Put/seqid=0 2024-11-21T11:15:30,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742208_1384 (size=12154) 2024-11-21T11:15:30,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:30,262 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e704b0bab36547ec8035ad72e5298fbd_d7b388d8fe831b9a9d75fdae52e5e157 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e704b0bab36547ec8035ad72e5298fbd_d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:30,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/c9e3137c19d04b41aa28387a50af0ae1, store: [table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:30,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/c9e3137c19d04b41aa28387a50af0ae1 is 175, key is test_row_0/A:col10/1732187728467/Put/seqid=0 2024-11-21T11:15:30,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742209_1385 (size=30955) 2024-11-21T11:15:30,277 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=108, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/c9e3137c19d04b41aa28387a50af0ae1 2024-11-21T11:15:30,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/7e4c794aa41546719c6044eed1635349 is 50, key is test_row_0/B:col10/1732187728467/Put/seqid=0 2024-11-21T11:15:30,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742210_1386 (size=12001) 2024-11-21T11:15:30,559 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/4d8665e1cd6f4222ba7eaa291565d67f as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/4d8665e1cd6f4222ba7eaa291565d67f 2024-11-21T11:15:30,563 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d7b388d8fe831b9a9d75fdae52e5e157/A of d7b388d8fe831b9a9d75fdae52e5e157 into 4d8665e1cd6f4222ba7eaa291565d67f(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:30,563 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7b388d8fe831b9a9d75fdae52e5e157: 2024-11-21T11:15:30,563 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., storeName=d7b388d8fe831b9a9d75fdae52e5e157/A, priority=12, startTime=1732187730117; duration=0sec 2024-11-21T11:15:30,563 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:30,563 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7b388d8fe831b9a9d75fdae52e5e157:A 2024-11-21T11:15:30,580 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/071d6a2516be44e08423e49a39ad4972 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/071d6a2516be44e08423e49a39ad4972 2024-11-21T11:15:30,584 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d7b388d8fe831b9a9d75fdae52e5e157/C of d7b388d8fe831b9a9d75fdae52e5e157 into 071d6a2516be44e08423e49a39ad4972(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:30,584 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7b388d8fe831b9a9d75fdae52e5e157: 2024-11-21T11:15:30,584 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., storeName=d7b388d8fe831b9a9d75fdae52e5e157/C, priority=12, startTime=1732187730117; duration=0sec 2024-11-21T11:15:30,585 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:30,585 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7b388d8fe831b9a9d75fdae52e5e157:C 2024-11-21T11:15:30,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:30,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:30,688 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/7e4c794aa41546719c6044eed1635349 2024-11-21T11:15:30,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/c3dbb55c6eac4ce699819d0565bbaa9e is 50, key is test_row_0/C:col10/1732187728467/Put/seqid=0 2024-11-21T11:15:30,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742211_1387 (size=12001) 2024-11-21T11:15:30,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732187730681, max timestamp=1732187730701Num uniques 859; 2024-11-21T11:15:30,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-21T11:15:30,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732187730681, max timestamp=1732187730701Num uniques 859; 2024-11-21T11:15:30,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-21T11:15:30,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732187730681, max timestamp=1732187730701Num uniques 859; 2024-11-21T11:15:30,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-21T11:15:30,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:30,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187790734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:30,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:30,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:30,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187790735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:30,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:30,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:30,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:30,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:30,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:30,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:30,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187790737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:30,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:30,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:30,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:30,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187790738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:30,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:30,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:30,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:30,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35324 deadline: 1732187790739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:30,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:30,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:30,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:30,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:30,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:30,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:30,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187790855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:30,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:30,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:30,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:30,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187790856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:30,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:30,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:30,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187790856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:30,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:30,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:30,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:30,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:30,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187790856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:30,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:30,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35324 deadline: 1732187790856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:31,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T11:15:31,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187791076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187791076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:31,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187791076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:31,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:31,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187791076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:31,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:31,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35324 deadline: 1732187791076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:31,102 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/c3dbb55c6eac4ce699819d0565bbaa9e 2024-11-21T11:15:31,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/c9e3137c19d04b41aa28387a50af0ae1 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/c9e3137c19d04b41aa28387a50af0ae1 2024-11-21T11:15:31,110 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/c9e3137c19d04b41aa28387a50af0ae1, entries=150, sequenceid=108, filesize=30.2 K 2024-11-21T11:15:31,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/7e4c794aa41546719c6044eed1635349 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/7e4c794aa41546719c6044eed1635349 2024-11-21T11:15:31,114 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/7e4c794aa41546719c6044eed1635349, entries=150, sequenceid=108, filesize=11.7 K 2024-11-21T11:15:31,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/c3dbb55c6eac4ce699819d0565bbaa9e as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/c3dbb55c6eac4ce699819d0565bbaa9e 2024-11-21T11:15:31,119 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/c3dbb55c6eac4ce699819d0565bbaa9e, entries=150, sequenceid=108, filesize=11.7 K 2024-11-21T11:15:31,120 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for d7b388d8fe831b9a9d75fdae52e5e157 in 874ms, sequenceid=108, compaction requested=false 2024-11-21T11:15:31,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for d7b388d8fe831b9a9d75fdae52e5e157: 2024-11-21T11:15:31,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:31,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-21T11:15:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-21T11:15:31,123 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-21T11:15:31,123 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2520 sec 2024-11-21T11:15:31,125 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.2570 sec 2024-11-21T11:15:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:31,404 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7b388d8fe831b9a9d75fdae52e5e157 3/3 column families, dataSize=147.60 KB heapSize=406.08 KB 2024-11-21T11:15:31,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=A 2024-11-21T11:15:31,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:15:31,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=B 2024-11-21T11:15:31,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:15:31,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=C 2024-11-21T11:15:31,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:15:31,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112161b9794f39a84a4b86038fd4251072b0_d7b388d8fe831b9a9d75fdae52e5e157 is 50, key is test_row_0/A:col10/1732187730738/Put/seqid=0 2024-11-21T11:15:31,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742212_1388 (size=14794) 2024-11-21T11:15:31,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187791409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:31,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:31,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35324 deadline: 1732187791410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:31,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:31,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187791413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:31,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:31,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187791425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:31,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:31,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:31,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187791426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:31,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:31,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:31,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187791535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:31,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:31,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35324 deadline: 1732187791535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:31,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:31,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:31,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187791535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:31,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:31,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187791536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:31,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:31,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187791536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:31,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:31,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187791755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:31,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:31,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35324 deadline: 1732187791756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:31,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:31,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187791756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:31,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187791756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:31,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187791756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:31,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:31,821 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:31,825 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112161b9794f39a84a4b86038fd4251072b0_d7b388d8fe831b9a9d75fdae52e5e157 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112161b9794f39a84a4b86038fd4251072b0_d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:31,826 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/cdcdf0c862da449bbec23b9e6f90ac91, store: [table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:31,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/cdcdf0c862da449bbec23b9e6f90ac91 is 175, key is test_row_0/A:col10/1732187730738/Put/seqid=0 2024-11-21T11:15:31,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742213_1389 (size=39749) 2024-11-21T11:15:32,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:32,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187792075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:32,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:32,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187792075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:32,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:32,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:32,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:32,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:32,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187792075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:32,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:32,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35324 deadline: 1732187792075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:32,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:32,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187792076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:32,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:32,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:32,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:32,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:32,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:32,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:32,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:32,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:32,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:32,231 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/cdcdf0c862da449bbec23b9e6f90ac91 2024-11-21T11:15:32,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/c2faca38b740440a8c32e5b2eb5dab8b is 50, key is test_row_0/B:col10/1732187730738/Put/seqid=0 2024-11-21T11:15:32,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742214_1390 (size=12151) 2024-11-21T11:15:32,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/c2faca38b740440a8c32e5b2eb5dab8b 2024-11-21T11:15:32,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/adc8a76a37cd40e79f7b59fb17151e65 is 50, key is test_row_0/C:col10/1732187730738/Put/seqid=0 2024-11-21T11:15:32,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742215_1391 (size=12151) 2024-11-21T11:15:32,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/adc8a76a37cd40e79f7b59fb17151e65 2024-11-21T11:15:32,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/cdcdf0c862da449bbec23b9e6f90ac91 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/cdcdf0c862da449bbec23b9e6f90ac91 2024-11-21T11:15:32,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/cdcdf0c862da449bbec23b9e6f90ac91, entries=200, sequenceid=145, filesize=38.8 K 2024-11-21T11:15:32,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/c2faca38b740440a8c32e5b2eb5dab8b as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/c2faca38b740440a8c32e5b2eb5dab8b 2024-11-21T11:15:32,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/c2faca38b740440a8c32e5b2eb5dab8b, entries=150, sequenceid=145, filesize=11.9 K 2024-11-21T11:15:32,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/adc8a76a37cd40e79f7b59fb17151e65 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/adc8a76a37cd40e79f7b59fb17151e65 2024-11-21T11:15:32,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/adc8a76a37cd40e79f7b59fb17151e65, entries=150, sequenceid=145, filesize=11.9 K 2024-11-21T11:15:32,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~406.03 KB/415776, currentSize=46.96 KB/48090 for d7b388d8fe831b9a9d75fdae52e5e157 in 950ms, sequenceid=145, compaction requested=true 2024-11-21T11:15:32,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7b388d8fe831b9a9d75fdae52e5e157: 2024-11-21T11:15:32,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7b388d8fe831b9a9d75fdae52e5e157:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:15:32,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:15:32,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7b388d8fe831b9a9d75fdae52e5e157:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:15:32,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T11:15:32,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7b388d8fe831b9a9d75fdae52e5e157:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:15:32,355 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:15:32,355 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:15:32,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:15:32,358 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:15:32,358 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): d7b388d8fe831b9a9d75fdae52e5e157/C is initiating minor compaction (all files) 2024-11-21T11:15:32,359 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7b388d8fe831b9a9d75fdae52e5e157/C in TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:32,359 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:15:32,359 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/071d6a2516be44e08423e49a39ad4972, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/c3dbb55c6eac4ce699819d0565bbaa9e, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/adc8a76a37cd40e79f7b59fb17151e65] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp, totalSize=35.4 K 2024-11-21T11:15:32,359 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): d7b388d8fe831b9a9d75fdae52e5e157/A is initiating minor compaction (all files) 2024-11-21T11:15:32,359 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7b388d8fe831b9a9d75fdae52e5e157/A in TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:32,359 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/4d8665e1cd6f4222ba7eaa291565d67f, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/c9e3137c19d04b41aa28387a50af0ae1, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/cdcdf0c862da449bbec23b9e6f90ac91] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp, totalSize=99.4 K 2024-11-21T11:15:32,359 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:32,359 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. files: [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/4d8665e1cd6f4222ba7eaa291565d67f, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/c9e3137c19d04b41aa28387a50af0ae1, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/cdcdf0c862da449bbec23b9e6f90ac91] 2024-11-21T11:15:32,362 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d8665e1cd6f4222ba7eaa291565d67f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732187728115 2024-11-21T11:15:32,362 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 071d6a2516be44e08423e49a39ad4972, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732187728115 2024-11-21T11:15:32,362 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting c3dbb55c6eac4ce699819d0565bbaa9e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732187728465 2024-11-21T11:15:32,362 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9e3137c19d04b41aa28387a50af0ae1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732187728465 2024-11-21T11:15:32,363 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdcdf0c862da449bbec23b9e6f90ac91, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732187730734 2024-11-21T11:15:32,363 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting adc8a76a37cd40e79f7b59fb17151e65, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732187730734 2024-11-21T11:15:32,383 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7b388d8fe831b9a9d75fdae52e5e157#C#compaction#329 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:15:32,383 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:32,383 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/67cdc31e879849fea4075fd914aa6726 is 50, key is test_row_0/C:col10/1732187730738/Put/seqid=0 2024-11-21T11:15:32,386 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121866cfaeb60d343099394f66bc1940afd_d7b388d8fe831b9a9d75fdae52e5e157 store=[table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:32,388 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121866cfaeb60d343099394f66bc1940afd_d7b388d8fe831b9a9d75fdae52e5e157, store=[table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:32,388 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121866cfaeb60d343099394f66bc1940afd_d7b388d8fe831b9a9d75fdae52e5e157 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:32,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742216_1392 (size=12391) 2024-11-21T11:15:32,395 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/67cdc31e879849fea4075fd914aa6726 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/67cdc31e879849fea4075fd914aa6726 2024-11-21T11:15:32,411 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7b388d8fe831b9a9d75fdae52e5e157/C of d7b388d8fe831b9a9d75fdae52e5e157 into 67cdc31e879849fea4075fd914aa6726(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:32,411 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7b388d8fe831b9a9d75fdae52e5e157: 2024-11-21T11:15:32,411 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., storeName=d7b388d8fe831b9a9d75fdae52e5e157/C, priority=13, startTime=1732187732355; duration=0sec 2024-11-21T11:15:32,411 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:15:32,411 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7b388d8fe831b9a9d75fdae52e5e157:C 2024-11-21T11:15:32,411 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:15:32,413 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:15:32,413 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): d7b388d8fe831b9a9d75fdae52e5e157/B is initiating minor compaction (all files) 2024-11-21T11:15:32,413 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7b388d8fe831b9a9d75fdae52e5e157/B in TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:32,414 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/60208f1d6cd64cedbbebd4733fae2b21, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/7e4c794aa41546719c6044eed1635349, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/c2faca38b740440a8c32e5b2eb5dab8b] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp, totalSize=35.4 K 2024-11-21T11:15:32,414 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 60208f1d6cd64cedbbebd4733fae2b21, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732187728115 2024-11-21T11:15:32,414 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e4c794aa41546719c6044eed1635349, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732187728465 2024-11-21T11:15:32,414 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting c2faca38b740440a8c32e5b2eb5dab8b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732187730734 2024-11-21T11:15:32,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742217_1393 (size=4469) 2024-11-21T11:15:32,427 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7b388d8fe831b9a9d75fdae52e5e157#B#compaction#331 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:15:32,427 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/d277542c8418402793d2066093dcc3c6 is 50, key is test_row_0/B:col10/1732187730738/Put/seqid=0 2024-11-21T11:15:32,431 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7b388d8fe831b9a9d75fdae52e5e157#A#compaction#330 average throughput is 0.52 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:15:32,431 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/9b0792e890824d71ba260dd444cb6ac0 is 175, key is test_row_0/A:col10/1732187730738/Put/seqid=0 2024-11-21T11:15:32,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742218_1394 (size=12391) 2024-11-21T11:15:32,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742219_1395 (size=31345) 2024-11-21T11:15:32,466 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/9b0792e890824d71ba260dd444cb6ac0 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/9b0792e890824d71ba260dd444cb6ac0 2024-11-21T11:15:32,470 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7b388d8fe831b9a9d75fdae52e5e157/A of d7b388d8fe831b9a9d75fdae52e5e157 into 9b0792e890824d71ba260dd444cb6ac0(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:32,470 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7b388d8fe831b9a9d75fdae52e5e157: 2024-11-21T11:15:32,470 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., storeName=d7b388d8fe831b9a9d75fdae52e5e157/A, priority=13, startTime=1732187732354; duration=0sec 2024-11-21T11:15:32,470 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:32,470 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7b388d8fe831b9a9d75fdae52e5e157:A 2024-11-21T11:15:32,472 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/d277542c8418402793d2066093dcc3c6 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/d277542c8418402793d2066093dcc3c6 2024-11-21T11:15:32,477 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7b388d8fe831b9a9d75fdae52e5e157/B of d7b388d8fe831b9a9d75fdae52e5e157 into d277542c8418402793d2066093dcc3c6(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:32,477 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7b388d8fe831b9a9d75fdae52e5e157: 2024-11-21T11:15:32,477 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., storeName=d7b388d8fe831b9a9d75fdae52e5e157/B, priority=13, startTime=1732187732355; duration=0sec 2024-11-21T11:15:32,477 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:32,477 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7b388d8fe831b9a9d75fdae52e5e157:B 2024-11-21T11:15:32,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:32,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7b388d8fe831b9a9d75fdae52e5e157 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-21T11:15:32,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=A 2024-11-21T11:15:32,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:32,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=B 2024-11-21T11:15:32,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:32,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=C 2024-11-21T11:15:32,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:32,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411210a39bb8a15d54749832801387d5a8599_d7b388d8fe831b9a9d75fdae52e5e157 is 50, key is test_row_0/A:col10/1732187732597/Put/seqid=0 2024-11-21T11:15:32,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742220_1396 (size=14794) 2024-11-21T11:15:32,614 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:32,620 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411210a39bb8a15d54749832801387d5a8599_d7b388d8fe831b9a9d75fdae52e5e157 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411210a39bb8a15d54749832801387d5a8599_d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:32,620 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/0a17d26b09b2477e9e2c000abd76c985, store: [table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:32,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/0a17d26b09b2477e9e2c000abd76c985 is 175, key is test_row_0/A:col10/1732187732597/Put/seqid=0 2024-11-21T11:15:32,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742221_1397 (size=39749) 2024-11-21T11:15:32,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:32,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187792639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:32,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732187732613, max timestamp=1732187732642Num uniques 859; 2024-11-21T11:15:32,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732187732613, max timestamp=1732187732642Num uniques 859; 2024-11-21T11:15:32,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T11:15:32,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T11:15:32,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:32,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:32,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:32,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732187732613, max timestamp=1732187732642Num uniques 859; 2024-11-21T11:15:32,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T11:15:32,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:32,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187792681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:32,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:32,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:32,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:32,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187792682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:32,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:32,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:32,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:32,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35324 deadline: 1732187792694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:32,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:32,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:32,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:32,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:32,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:32,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:32,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187792695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:32,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:32,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:32,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:32,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:32,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187792755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:32,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:32,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:32,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:32,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:32,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187792805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:32,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:32,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:32,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:32,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:32,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187792805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:32,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:32,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35324 deadline: 1732187792805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:32,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:32,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:32,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:32,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:32,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:32,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:32,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:32,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187792815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:32,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:32,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:32,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:32,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:32,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187792976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:32,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:32,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:32,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:33,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T11:15:33,026 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:15:33,026 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:15:33,026 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/0a17d26b09b2477e9e2c000abd76c985 2024-11-21T11:15:33,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-21T11:15:33,028 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:15:33,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T11:15:33,029 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:15:33,029 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:15:33,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:33,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187793025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:33,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:33,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:33,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187793026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:33,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:33,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:33,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:33,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35324 deadline: 1732187793026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:33,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/ec47583ea6e1465cb695495ddc47ca0e is 50, key is test_row_0/B:col10/1732187732597/Put/seqid=0 2024-11-21T11:15:33,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:33,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:33,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:33,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:33,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:33,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:33,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742222_1398 (size=12151) 2024-11-21T11:15:33,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:33,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187793035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:33,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:33,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:33,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:33,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T11:15:33,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T11:15:33,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:33,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:33,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:33,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:33,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:33,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:33,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:33,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187793295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:33,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:33,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:33,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:33,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T11:15:33,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:33,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:33,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:33,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:33,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:33,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:33,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T11:15:33,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:33,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35324 deadline: 1732187793346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:33,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:33,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187793346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:33,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:33,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:33,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:33,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:33,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187793346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:33,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:33,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:33,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:33,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:33,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:33,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:33,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:33,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187793355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:33,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:33,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:33,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:33,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/ec47583ea6e1465cb695495ddc47ca0e 2024-11-21T11:15:33,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/6fea7e465ca14b56881f25b9555e6772 is 50, key is test_row_0/C:col10/1732187732597/Put/seqid=0 2024-11-21T11:15:33,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742223_1399 (size=12151) 2024-11-21T11:15:33,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/6fea7e465ca14b56881f25b9555e6772 2024-11-21T11:15:33,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/0a17d26b09b2477e9e2c000abd76c985 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/0a17d26b09b2477e9e2c000abd76c985 2024-11-21T11:15:33,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/0a17d26b09b2477e9e2c000abd76c985, entries=200, sequenceid=159, filesize=38.8 K 2024-11-21T11:15:33,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/ec47583ea6e1465cb695495ddc47ca0e as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/ec47583ea6e1465cb695495ddc47ca0e 2024-11-21T11:15:33,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/ec47583ea6e1465cb695495ddc47ca0e, entries=150, sequenceid=159, filesize=11.9 K 2024-11-21T11:15:33,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/6fea7e465ca14b56881f25b9555e6772 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/6fea7e465ca14b56881f25b9555e6772 2024-11-21T11:15:33,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/6fea7e465ca14b56881f25b9555e6772, entries=150, sequenceid=159, filesize=11.9 K 2024-11-21T11:15:33,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for d7b388d8fe831b9a9d75fdae52e5e157 in 889ms, sequenceid=159, compaction requested=false 2024-11-21T11:15:33,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7b388d8fe831b9a9d75fdae52e5e157: 2024-11-21T11:15:33,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T11:15:33,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:33,489 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing d7b388d8fe831b9a9d75fdae52e5e157 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-21T11:15:33,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=A 2024-11-21T11:15:33,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:15:33,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=B 2024-11-21T11:15:33,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:15:33,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=C 2024-11-21T11:15:33,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:15:33,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112180e74cbe7f2642eeaa1eecd49e2b9ace_d7b388d8fe831b9a9d75fdae52e5e157 is 50, key is test_row_0/A:col10/1732187732692/Put/seqid=0 2024-11-21T11:15:33,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742224_1400 (size=12304) 2024-11-21T11:15:33,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:33,503 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112180e74cbe7f2642eeaa1eecd49e2b9ace_d7b388d8fe831b9a9d75fdae52e5e157 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112180e74cbe7f2642eeaa1eecd49e2b9ace_d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:33,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/53665af587d3493fb67d98d0dee0f986, store: [table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:33,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/53665af587d3493fb67d98d0dee0f986 is 175, key is test_row_0/A:col10/1732187732692/Put/seqid=0 2024-11-21T11:15:33,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742225_1401 (size=31105) 2024-11-21T11:15:33,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T11:15:33,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:33,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:33,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:33,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187793866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:33,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:33,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:33,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:33,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:33,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187793872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:33,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:33,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:33,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:33,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:33,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35324 deadline: 1732187793875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:33,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:33,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:33,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:33,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:33,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187793875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:33,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:33,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:33,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:33,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:33,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187793875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:33,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:33,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:33,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:33,914 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/53665af587d3493fb67d98d0dee0f986 2024-11-21T11:15:33,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/83816a299e174feca525c362fb9cbb4c is 50, key is test_row_0/B:col10/1732187732692/Put/seqid=0 2024-11-21T11:15:33,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742226_1402 (size=12151) 2024-11-21T11:15:33,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:33,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187793975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:33,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:33,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:33,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:33,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:33,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187793985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:33,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:33,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:33,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:34,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T11:15:34,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:34,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187794185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:34,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:34,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:34,196 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:34,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:34,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187794205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:34,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:34,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:34,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:34,329 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/83816a299e174feca525c362fb9cbb4c 2024-11-21T11:15:34,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/b80aa51694554bc1bc42021d85440198 is 50, key is test_row_0/C:col10/1732187732692/Put/seqid=0 2024-11-21T11:15:34,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742227_1403 (size=12151) 2024-11-21T11:15:34,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:34,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187794506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:34,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:34,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:34,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:34,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:34,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187794525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:34,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:34,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:34,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:34,744 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/b80aa51694554bc1bc42021d85440198 2024-11-21T11:15:34,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/53665af587d3493fb67d98d0dee0f986 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/53665af587d3493fb67d98d0dee0f986 2024-11-21T11:15:34,752 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/53665af587d3493fb67d98d0dee0f986, entries=150, sequenceid=194, filesize=30.4 K 2024-11-21T11:15:34,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/83816a299e174feca525c362fb9cbb4c as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/83816a299e174feca525c362fb9cbb4c 2024-11-21T11:15:34,756 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/83816a299e174feca525c362fb9cbb4c, entries=150, sequenceid=194, filesize=11.9 K 2024-11-21T11:15:34,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/b80aa51694554bc1bc42021d85440198 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/b80aa51694554bc1bc42021d85440198 2024-11-21T11:15:34,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,760 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/b80aa51694554bc1bc42021d85440198, entries=150, sequenceid=194, filesize=11.9 K 2024-11-21T11:15:34,761 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for d7b388d8fe831b9a9d75fdae52e5e157 in 1271ms, sequenceid=194, compaction requested=true 2024-11-21T11:15:34,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for d7b388d8fe831b9a9d75fdae52e5e157: 2024-11-21T11:15:34,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:34,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-21T11:15:34,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-21T11:15:34,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,764 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-21T11:15:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,764 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7330 sec 2024-11-21T11:15:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,771 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.7390 sec 2024-11-21T11:15:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:34,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7b388d8fe831b9a9d75fdae52e5e157 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-21T11:15:34,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=A 2024-11-21T11:15:34,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:34,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=B 2024-11-21T11:15:34,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:34,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=C 2024-11-21T11:15:34,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:34,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411217c6b5efbf88b41b897536d37aa602e03_d7b388d8fe831b9a9d75fdae52e5e157 is 50, key is test_row_0/A:col10/1732187734908/Put/seqid=0 2024-11-21T11:15:34,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742228_1404 (size=12304) 2024-11-21T11:15:34,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,932 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,936 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411217c6b5efbf88b41b897536d37aa602e03_d7b388d8fe831b9a9d75fdae52e5e157 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411217c6b5efbf88b41b897536d37aa602e03_d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:34,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,940 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/dd96013b62a04431bf351017c492d1e3, store: [table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:34,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/dd96013b62a04431bf351017c492d1e3 is 175, key is test_row_0/A:col10/1732187734908/Put/seqid=0 2024-11-21T11:15:34,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742229_1405 (size=31101) 2024-11-21T11:15:34,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:34,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187794977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:34,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:34,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187794982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:34,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732187734961, max timestamp=1732187734988Num uniques 859; 2024-11-21T11:15:34,992 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T11:15:34,992 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732187734961, max timestamp=1732187734988Num uniques 859; 2024-11-21T11:15:34,992 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732187734961, max timestamp=1732187734988Num uniques 859; 2024-11-21T11:15:34,992 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T11:15:34,992 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T11:15:34,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:34,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:34,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:34,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:34,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:34,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:35,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:35,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187795070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:35,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:35,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:35,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:35,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:35,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187795073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:35,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:35,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35324 deadline: 1732187795074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:35,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:35,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:35,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:35,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:35,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:35,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:35,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:35,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187795095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:35,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:35,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187795095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:35,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:35,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:35,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:35,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T11:15:35,176 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:15:35,177 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:15:35,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-21T11:15:35,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T11:15:35,179 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:15:35,179 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:15:35,179 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:15:35,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187795185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:35,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:35,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187795205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:35,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:35,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35324 deadline: 1732187795205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:35,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:35,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:35,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:35,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:35,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:35,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:35,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T11:15:35,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:35,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187795305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:35,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:35,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187795306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:35,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T11:15:35,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:35,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:35,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:35,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:35,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:35,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:35,344 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=205, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/dd96013b62a04431bf351017c492d1e3 2024-11-21T11:15:35,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/09759359e5f642c3bb58ab8e375b7a7b is 50, key is test_row_0/B:col10/1732187734908/Put/seqid=0 2024-11-21T11:15:35,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742230_1406 (size=9757) 2024-11-21T11:15:35,360 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/09759359e5f642c3bb58ab8e375b7a7b 2024-11-21T11:15:35,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/24f65327a8fc4381adb6f61130c50de0 is 50, key is test_row_0/C:col10/1732187734908/Put/seqid=0 2024-11-21T11:15:35,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742231_1407 (size=9757) 2024-11-21T11:15:35,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:35,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187795405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:35,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:35,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:35,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:35,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:35,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187795426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:35,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:35,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35324 deadline: 1732187795426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:35,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:35,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:35,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:35,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T11:15:35,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:35,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:35,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:35,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:35,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:35,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:35,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T11:15:35,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:35,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187795626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:35,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:35,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:35,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:35,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:35,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187795626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:35,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:35,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:35,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:35,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T11:15:35,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:35,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:35,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:35,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:35,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:35,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:35,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187795726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:35,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:35,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:35,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:35,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187795745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:35,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:35,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35324 deadline: 1732187795745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:35,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:35,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:35,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:35,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:35,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:35,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/24f65327a8fc4381adb6f61130c50de0 2024-11-21T11:15:35,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/dd96013b62a04431bf351017c492d1e3 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/dd96013b62a04431bf351017c492d1e3 2024-11-21T11:15:35,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/dd96013b62a04431bf351017c492d1e3, entries=150, sequenceid=205, filesize=30.4 K 2024-11-21T11:15:35,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/09759359e5f642c3bb58ab8e375b7a7b as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/09759359e5f642c3bb58ab8e375b7a7b 2024-11-21T11:15:35,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/09759359e5f642c3bb58ab8e375b7a7b, entries=100, sequenceid=205, filesize=9.5 K 2024-11-21T11:15:35,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/24f65327a8fc4381adb6f61130c50de0 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/24f65327a8fc4381adb6f61130c50de0 2024-11-21T11:15:35,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/24f65327a8fc4381adb6f61130c50de0, entries=100, sequenceid=205, filesize=9.5 K 2024-11-21T11:15:35,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for d7b388d8fe831b9a9d75fdae52e5e157 in 879ms, sequenceid=205, compaction requested=true 2024-11-21T11:15:35,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7b388d8fe831b9a9d75fdae52e5e157: 2024-11-21T11:15:35,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7b388d8fe831b9a9d75fdae52e5e157:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:15:35,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:35,789 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T11:15:35,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7b388d8fe831b9a9d75fdae52e5e157:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:15:35,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:35,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7b388d8fe831b9a9d75fdae52e5e157:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:15:35,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:15:35,789 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T11:15:35,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T11:15:35,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:35,790 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing d7b388d8fe831b9a9d75fdae52e5e157 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-21T11:15:35,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=A 2024-11-21T11:15:35,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:15:35,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=B 2024-11-21T11:15:35,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:15:35,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=C 2024-11-21T11:15:35,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:15:35,791 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133300 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T11:15:35,791 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): d7b388d8fe831b9a9d75fdae52e5e157/A is initiating minor compaction (all files) 2024-11-21T11:15:35,791 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T11:15:35,791 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7b388d8fe831b9a9d75fdae52e5e157/A in TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:35,791 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): d7b388d8fe831b9a9d75fdae52e5e157/B is initiating minor compaction (all files) 2024-11-21T11:15:35,791 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/9b0792e890824d71ba260dd444cb6ac0, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/0a17d26b09b2477e9e2c000abd76c985, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/53665af587d3493fb67d98d0dee0f986, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/dd96013b62a04431bf351017c492d1e3] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp, totalSize=130.2 K 2024-11-21T11:15:35,791 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7b388d8fe831b9a9d75fdae52e5e157/B in TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:35,792 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:35,792 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/d277542c8418402793d2066093dcc3c6, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/ec47583ea6e1465cb695495ddc47ca0e, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/83816a299e174feca525c362fb9cbb4c, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/09759359e5f642c3bb58ab8e375b7a7b] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp, totalSize=45.4 K 2024-11-21T11:15:35,792 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. files: [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/9b0792e890824d71ba260dd444cb6ac0, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/0a17d26b09b2477e9e2c000abd76c985, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/53665af587d3493fb67d98d0dee0f986, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/dd96013b62a04431bf351017c492d1e3] 2024-11-21T11:15:35,792 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting d277542c8418402793d2066093dcc3c6, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732187730734 2024-11-21T11:15:35,792 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b0792e890824d71ba260dd444cb6ac0, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732187730734 2024-11-21T11:15:35,794 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a17d26b09b2477e9e2c000abd76c985, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732187731408 2024-11-21T11:15:35,794 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting ec47583ea6e1465cb695495ddc47ca0e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732187731408 2024-11-21T11:15:35,794 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 83816a299e174feca525c362fb9cbb4c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732187732680 2024-11-21T11:15:35,794 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53665af587d3493fb67d98d0dee0f986, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732187732680 2024-11-21T11:15:35,794 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 09759359e5f642c3bb58ab8e375b7a7b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732187733864 2024-11-21T11:15:35,794 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd96013b62a04431bf351017c492d1e3, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732187733864 2024-11-21T11:15:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T11:15:35,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f0abd431549b4799b6db2a4e220936a6_d7b388d8fe831b9a9d75fdae52e5e157 is 50, key is test_row_0/A:col10/1732187735071/Put/seqid=0 2024-11-21T11:15:35,807 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7b388d8fe831b9a9d75fdae52e5e157#B#compaction#342 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:15:35,808 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/81ac939b1ff140529ff244b4a3d987eb is 50, key is test_row_0/B:col10/1732187734908/Put/seqid=0 2024-11-21T11:15:35,808 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:35,814 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121d77a09bac2cd437e839e884016ac6ff4_d7b388d8fe831b9a9d75fdae52e5e157 store=[table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:35,817 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121d77a09bac2cd437e839e884016ac6ff4_d7b388d8fe831b9a9d75fdae52e5e157, store=[table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:35,818 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d77a09bac2cd437e839e884016ac6ff4_d7b388d8fe831b9a9d75fdae52e5e157 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:35,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742233_1409 (size=12527) 2024-11-21T11:15:35,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742232_1408 (size=12304) 2024-11-21T11:15:35,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742234_1410 (size=4469) 2024-11-21T11:15:36,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:36,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:36,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:36,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187796185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:36,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:36,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:36,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:36,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:36,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187796195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:36,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:36,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:36,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:36,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:36,227 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f0abd431549b4799b6db2a4e220936a6_d7b388d8fe831b9a9d75fdae52e5e157 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f0abd431549b4799b6db2a4e220936a6_d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:36,227 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/81ac939b1ff140529ff244b4a3d987eb as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/81ac939b1ff140529ff244b4a3d987eb 2024-11-21T11:15:36,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/8f80b3425ce3490d86282ef746572ad7, store: [table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:36,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/8f80b3425ce3490d86282ef746572ad7 is 175, key is test_row_0/A:col10/1732187735071/Put/seqid=0 2024-11-21T11:15:36,231 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7b388d8fe831b9a9d75fdae52e5e157#A#compaction#343 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:15:36,232 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/b33f350a8c0b4eb69baf00c4cecd9319 is 175, key is test_row_0/A:col10/1732187734908/Put/seqid=0 2024-11-21T11:15:36,233 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d7b388d8fe831b9a9d75fdae52e5e157/B of d7b388d8fe831b9a9d75fdae52e5e157 into 81ac939b1ff140529ff244b4a3d987eb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:36,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742235_1411 (size=31105) 2024-11-21T11:15:36,233 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7b388d8fe831b9a9d75fdae52e5e157: 2024-11-21T11:15:36,233 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., storeName=d7b388d8fe831b9a9d75fdae52e5e157/B, priority=12, startTime=1732187735789; duration=0sec 2024-11-21T11:15:36,233 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:15:36,233 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7b388d8fe831b9a9d75fdae52e5e157:B 2024-11-21T11:15:36,233 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T11:15:36,235 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T11:15:36,235 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): d7b388d8fe831b9a9d75fdae52e5e157/C is initiating minor compaction (all files) 2024-11-21T11:15:36,235 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7b388d8fe831b9a9d75fdae52e5e157/C in TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:36,235 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/67cdc31e879849fea4075fd914aa6726, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/6fea7e465ca14b56881f25b9555e6772, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/b80aa51694554bc1bc42021d85440198, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/24f65327a8fc4381adb6f61130c50de0] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp, totalSize=45.4 K 2024-11-21T11:15:36,236 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 67cdc31e879849fea4075fd914aa6726, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732187730734 2024-11-21T11:15:36,236 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fea7e465ca14b56881f25b9555e6772, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732187731408 2024-11-21T11:15:36,237 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting b80aa51694554bc1bc42021d85440198, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732187732680 2024-11-21T11:15:36,237 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 24f65327a8fc4381adb6f61130c50de0, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732187733864 2024-11-21T11:15:36,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742236_1412 (size=31588) 2024-11-21T11:15:36,248 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7b388d8fe831b9a9d75fdae52e5e157#C#compaction#344 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:15:36,248 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/2ebaf9635f744700b05fcd6ea0c54040 is 50, key is test_row_0/C:col10/1732187734908/Put/seqid=0 2024-11-21T11:15:36,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742237_1413 (size=12527) 2024-11-21T11:15:36,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:36,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187796245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:36,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:36,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:36,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:36,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:36,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187796265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:36,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:36,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:36,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:36,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:36,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35324 deadline: 1732187796265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:36,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:36,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:36,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:36,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:36,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187796305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:36,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:36,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:36,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:36,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T11:15:36,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:36,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187796315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:36,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:36,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187796516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:36,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:36,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:36,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:36,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:36,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187796535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:36,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:36,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:36,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:36,633 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=240, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/8f80b3425ce3490d86282ef746572ad7 2024-11-21T11:15:36,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/c469dd98db9446cbb676e15a962f2511 is 50, key is test_row_0/B:col10/1732187735071/Put/seqid=0 2024-11-21T11:15:36,643 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/b33f350a8c0b4eb69baf00c4cecd9319 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/b33f350a8c0b4eb69baf00c4cecd9319 2024-11-21T11:15:36,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742238_1414 (size=12151) 2024-11-21T11:15:36,648 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d7b388d8fe831b9a9d75fdae52e5e157/A of d7b388d8fe831b9a9d75fdae52e5e157 into b33f350a8c0b4eb69baf00c4cecd9319(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:36,648 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7b388d8fe831b9a9d75fdae52e5e157: 2024-11-21T11:15:36,648 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., storeName=d7b388d8fe831b9a9d75fdae52e5e157/A, priority=12, startTime=1732187735789; duration=0sec 2024-11-21T11:15:36,648 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:36,648 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7b388d8fe831b9a9d75fdae52e5e157:A 2024-11-21T11:15:36,656 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T11:15:36,657 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/2ebaf9635f744700b05fcd6ea0c54040 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/2ebaf9635f744700b05fcd6ea0c54040 2024-11-21T11:15:36,660 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d7b388d8fe831b9a9d75fdae52e5e157/C of d7b388d8fe831b9a9d75fdae52e5e157 into 2ebaf9635f744700b05fcd6ea0c54040(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:36,660 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7b388d8fe831b9a9d75fdae52e5e157: 2024-11-21T11:15:36,661 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., storeName=d7b388d8fe831b9a9d75fdae52e5e157/C, priority=12, startTime=1732187735789; duration=0sec 2024-11-21T11:15:36,661 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:36,661 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7b388d8fe831b9a9d75fdae52e5e157:C 2024-11-21T11:15:36,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:36,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187796825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:36,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:36,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187796855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:36,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:36,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:36,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:37,046 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/c469dd98db9446cbb676e15a962f2511 2024-11-21T11:15:37,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/1bbee339d520404a9c8cd20628ced9da is 50, key is test_row_0/C:col10/1732187735071/Put/seqid=0 2024-11-21T11:15:37,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742239_1415 (size=12151) 2024-11-21T11:15:37,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187797275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:37,280 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_2' in d7b388d8fe831b9a9d75fdae52e5e157 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:15:37,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:37,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:37,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:37,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187797286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:37,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35324 deadline: 1732187797286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:37,295 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_0' in d7b388d8fe831b9a9d75fdae52e5e157 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:15:37,295 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_0' in d7b388d8fe831b9a9d75fdae52e5e157 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:15:37,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:37,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:37,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:37,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:37,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:37,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T11:15:37,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:37,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187797345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:37,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:37,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:37,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:37,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:37,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187797376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:37,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:37,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:37,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:37,458 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/1bbee339d520404a9c8cd20628ced9da 2024-11-21T11:15:37,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/8f80b3425ce3490d86282ef746572ad7 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/8f80b3425ce3490d86282ef746572ad7 2024-11-21T11:15:37,466 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/8f80b3425ce3490d86282ef746572ad7, entries=150, sequenceid=240, filesize=30.4 K 2024-11-21T11:15:37,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/c469dd98db9446cbb676e15a962f2511 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/c469dd98db9446cbb676e15a962f2511 2024-11-21T11:15:37,470 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/c469dd98db9446cbb676e15a962f2511, entries=150, sequenceid=240, filesize=11.9 K 2024-11-21T11:15:37,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/1bbee339d520404a9c8cd20628ced9da as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/1bbee339d520404a9c8cd20628ced9da 2024-11-21T11:15:37,474 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/1bbee339d520404a9c8cd20628ced9da, entries=150, sequenceid=240, filesize=11.9 K 2024-11-21T11:15:37,475 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for d7b388d8fe831b9a9d75fdae52e5e157 in 1685ms, sequenceid=240, compaction requested=false 2024-11-21T11:15:37,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for d7b388d8fe831b9a9d75fdae52e5e157: 2024-11-21T11:15:37,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:37,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-21T11:15:37,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-21T11:15:37,478 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-21T11:15:37,478 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2980 sec 2024-11-21T11:15:37,481 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.3020 sec 2024-11-21T11:15:38,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:38,368 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7b388d8fe831b9a9d75fdae52e5e157 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-21T11:15:38,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=A 2024-11-21T11:15:38,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:38,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=B 2024-11-21T11:15:38,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:38,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=C 2024-11-21T11:15:38,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:38,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112101c1c51e85b649cabdd82255461ae816_d7b388d8fe831b9a9d75fdae52e5e157 is 50, key is test_row_0/A:col10/1732187736194/Put/seqid=0 2024-11-21T11:15:38,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742240_1416 (size=14794) 2024-11-21T11:15:38,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:38,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187798456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:38,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:38,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:38,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:38,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187798457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:38,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:38,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:38,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:38,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:38,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:38,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187798565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:38,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:38,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:38,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187798565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:38,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:38,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:38,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:38,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:38,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:38,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:38,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187798775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:38,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:38,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187798775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:38,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:38,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:38,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:38,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:38,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:38,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:38,788 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:38,791 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112101c1c51e85b649cabdd82255461ae816_d7b388d8fe831b9a9d75fdae52e5e157 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112101c1c51e85b649cabdd82255461ae816_d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:38,792 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/35be6f145bfc45b58ec8b4dc6ab41fcd, store: [table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:38,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/35be6f145bfc45b58ec8b4dc6ab41fcd is 175, key is test_row_0/A:col10/1732187736194/Put/seqid=0 2024-11-21T11:15:38,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742241_1417 (size=39749) 2024-11-21T11:15:39,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:39,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187799095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:39,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:39,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187799095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:39,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:39,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:39,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:39,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:39,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:39,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:39,197 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=255, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/35be6f145bfc45b58ec8b4dc6ab41fcd 2024-11-21T11:15:39,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/d69cabe4b8f4411f831e226acb7a005a is 50, key is test_row_0/B:col10/1732187736194/Put/seqid=0 2024-11-21T11:15:39,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742242_1418 (size=12151) 2024-11-21T11:15:39,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:39,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187799295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:39,305 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_2' in d7b388d8fe831b9a9d75fdae52e5e157 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:15:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:39,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35324 deadline: 1732187799315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:39,318 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_0' in d7b388d8fe831b9a9d75fdae52e5e157 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:15:39,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:39,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:39,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:39,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:39,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187799315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:39,326 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_0' in d7b388d8fe831b9a9d75fdae52e5e157 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4254 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:15:39,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:39,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:39,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:39,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T11:15:39,345 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:15:39,345 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:15:39,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-21T11:15:39,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T11:15:39,347 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:15:39,348 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:15:39,348 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:15:39,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T11:15:39,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-21T11:15:39,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:39,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:39,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:39,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:39,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:39,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:39,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/d69cabe4b8f4411f831e226acb7a005a 2024-11-21T11:15:39,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/46f25cb9b233448fa9c9c353fb01de10 is 50, key is test_row_0/C:col10/1732187736194/Put/seqid=0 2024-11-21T11:15:39,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742243_1419 (size=12151) 2024-11-21T11:15:39,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:39,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187799615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:39,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:39,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187799615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:39,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:39,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:39,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:39,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:39,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:39,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:39,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-21T11:15:39,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:39,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:39,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:39,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:39,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:39,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:39,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T11:15:39,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-21T11:15:39,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:39,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:39,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:39,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:39,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:39,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:39,902 WARN [regionserver/7b462513bfc2:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-21T11:15:39,902 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/03e9bdbaa36d42a2bc63fbcbde9fb99c, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/702ccec7b0164c19b923d7054f78a588, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/dd9263d6dc064e66ace9706ff217f9c5, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/2e8d1607146a4c959aebf7b00baa76e6, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/4d8665e1cd6f4222ba7eaa291565d67f, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/c9e3137c19d04b41aa28387a50af0ae1, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/cdcdf0c862da449bbec23b9e6f90ac91, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/9b0792e890824d71ba260dd444cb6ac0, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/0a17d26b09b2477e9e2c000abd76c985, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/53665af587d3493fb67d98d0dee0f986, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/dd96013b62a04431bf351017c492d1e3] to archive 2024-11-21T11:15:39,903 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T11:15:39,904 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/03e9bdbaa36d42a2bc63fbcbde9fb99c to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/03e9bdbaa36d42a2bc63fbcbde9fb99c 2024-11-21T11:15:39,905 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/702ccec7b0164c19b923d7054f78a588 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/702ccec7b0164c19b923d7054f78a588 2024-11-21T11:15:39,906 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/dd9263d6dc064e66ace9706ff217f9c5 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/dd9263d6dc064e66ace9706ff217f9c5 2024-11-21T11:15:39,907 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/2e8d1607146a4c959aebf7b00baa76e6 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/2e8d1607146a4c959aebf7b00baa76e6 2024-11-21T11:15:39,908 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/4d8665e1cd6f4222ba7eaa291565d67f to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/4d8665e1cd6f4222ba7eaa291565d67f 2024-11-21T11:15:39,909 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/c9e3137c19d04b41aa28387a50af0ae1 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/c9e3137c19d04b41aa28387a50af0ae1 2024-11-21T11:15:39,910 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/cdcdf0c862da449bbec23b9e6f90ac91 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/cdcdf0c862da449bbec23b9e6f90ac91 2024-11-21T11:15:39,911 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/9b0792e890824d71ba260dd444cb6ac0 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/9b0792e890824d71ba260dd444cb6ac0 2024-11-21T11:15:39,912 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/0a17d26b09b2477e9e2c000abd76c985 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/0a17d26b09b2477e9e2c000abd76c985 2024-11-21T11:15:39,913 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/53665af587d3493fb67d98d0dee0f986 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/53665af587d3493fb67d98d0dee0f986 2024-11-21T11:15:39,914 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/dd96013b62a04431bf351017c492d1e3 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/dd96013b62a04431bf351017c492d1e3 2024-11-21T11:15:39,916 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/449de254c9084c4da0d752bca32b6fb1, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/8ee27da56edc4df1a1e6a6cb9f88ff4a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/6f8f0d43078644f0b81175f932bd6c24, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/60208f1d6cd64cedbbebd4733fae2b21, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/f1177a52934c4acc98fbffab3d2ae9c5, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/7e4c794aa41546719c6044eed1635349, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/d277542c8418402793d2066093dcc3c6, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/c2faca38b740440a8c32e5b2eb5dab8b, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/ec47583ea6e1465cb695495ddc47ca0e, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/83816a299e174feca525c362fb9cbb4c, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/09759359e5f642c3bb58ab8e375b7a7b] to archive 2024-11-21T11:15:39,917 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T11:15:39,918 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/449de254c9084c4da0d752bca32b6fb1 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/449de254c9084c4da0d752bca32b6fb1 2024-11-21T11:15:39,920 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/8ee27da56edc4df1a1e6a6cb9f88ff4a to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/8ee27da56edc4df1a1e6a6cb9f88ff4a 2024-11-21T11:15:39,921 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/6f8f0d43078644f0b81175f932bd6c24 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/6f8f0d43078644f0b81175f932bd6c24 2024-11-21T11:15:39,922 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/60208f1d6cd64cedbbebd4733fae2b21 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/60208f1d6cd64cedbbebd4733fae2b21 2024-11-21T11:15:39,922 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/f1177a52934c4acc98fbffab3d2ae9c5 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/f1177a52934c4acc98fbffab3d2ae9c5 2024-11-21T11:15:39,923 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/7e4c794aa41546719c6044eed1635349 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/7e4c794aa41546719c6044eed1635349 2024-11-21T11:15:39,924 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/d277542c8418402793d2066093dcc3c6 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/d277542c8418402793d2066093dcc3c6 2024-11-21T11:15:39,925 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/c2faca38b740440a8c32e5b2eb5dab8b to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/c2faca38b740440a8c32e5b2eb5dab8b 2024-11-21T11:15:39,926 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/ec47583ea6e1465cb695495ddc47ca0e to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/ec47583ea6e1465cb695495ddc47ca0e 2024-11-21T11:15:39,927 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/83816a299e174feca525c362fb9cbb4c to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/83816a299e174feca525c362fb9cbb4c 2024-11-21T11:15:39,928 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/09759359e5f642c3bb58ab8e375b7a7b to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/09759359e5f642c3bb58ab8e375b7a7b 2024-11-21T11:15:39,930 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/50b7050b35d74bd88ec83ec87baea952, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/b6409ddd584a42069a51689f961a61f2, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/c42292580e0b4db5b37f820741b37e62, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/071d6a2516be44e08423e49a39ad4972, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/97e1be61b562412e937da3e24363d4b0, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/c3dbb55c6eac4ce699819d0565bbaa9e, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/67cdc31e879849fea4075fd914aa6726, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/adc8a76a37cd40e79f7b59fb17151e65, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/6fea7e465ca14b56881f25b9555e6772, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/b80aa51694554bc1bc42021d85440198, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/24f65327a8fc4381adb6f61130c50de0] to archive 2024-11-21T11:15:39,931 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T11:15:39,932 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/50b7050b35d74bd88ec83ec87baea952 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/50b7050b35d74bd88ec83ec87baea952 2024-11-21T11:15:39,933 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/b6409ddd584a42069a51689f961a61f2 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/b6409ddd584a42069a51689f961a61f2 2024-11-21T11:15:39,934 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/c42292580e0b4db5b37f820741b37e62 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/c42292580e0b4db5b37f820741b37e62 2024-11-21T11:15:39,935 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/071d6a2516be44e08423e49a39ad4972 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/071d6a2516be44e08423e49a39ad4972 2024-11-21T11:15:39,936 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/97e1be61b562412e937da3e24363d4b0 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/97e1be61b562412e937da3e24363d4b0 2024-11-21T11:15:39,937 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/c3dbb55c6eac4ce699819d0565bbaa9e to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/c3dbb55c6eac4ce699819d0565bbaa9e 2024-11-21T11:15:39,938 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/67cdc31e879849fea4075fd914aa6726 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/67cdc31e879849fea4075fd914aa6726 2024-11-21T11:15:39,939 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/adc8a76a37cd40e79f7b59fb17151e65 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/adc8a76a37cd40e79f7b59fb17151e65 2024-11-21T11:15:39,940 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/6fea7e465ca14b56881f25b9555e6772 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/6fea7e465ca14b56881f25b9555e6772 2024-11-21T11:15:39,941 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/b80aa51694554bc1bc42021d85440198 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/b80aa51694554bc1bc42021d85440198 2024-11-21T11:15:39,942 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/24f65327a8fc4381adb6f61130c50de0 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/24f65327a8fc4381adb6f61130c50de0 2024-11-21T11:15:39,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-21T11:15:39,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:39,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:39,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:39,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:39,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:39,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T11:15:40,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/46f25cb9b233448fa9c9c353fb01de10 2024-11-21T11:15:40,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/35be6f145bfc45b58ec8b4dc6ab41fcd as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/35be6f145bfc45b58ec8b4dc6ab41fcd 2024-11-21T11:15:40,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/35be6f145bfc45b58ec8b4dc6ab41fcd, entries=200, sequenceid=255, filesize=38.8 K 2024-11-21T11:15:40,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/d69cabe4b8f4411f831e226acb7a005a as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/d69cabe4b8f4411f831e226acb7a005a 2024-11-21T11:15:40,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/d69cabe4b8f4411f831e226acb7a005a, entries=150, sequenceid=255, filesize=11.9 K 2024-11-21T11:15:40,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/46f25cb9b233448fa9c9c353fb01de10 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/46f25cb9b233448fa9c9c353fb01de10 2024-11-21T11:15:40,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/46f25cb9b233448fa9c9c353fb01de10, entries=150, sequenceid=255, filesize=11.9 K 2024-11-21T11:15:40,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d7b388d8fe831b9a9d75fdae52e5e157 in 1672ms, sequenceid=255, compaction requested=true 2024-11-21T11:15:40,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7b388d8fe831b9a9d75fdae52e5e157: 2024-11-21T11:15:40,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7b388d8fe831b9a9d75fdae52e5e157:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:15:40,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:40,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7b388d8fe831b9a9d75fdae52e5e157:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:15:40,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:40,040 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:15:40,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7b388d8fe831b9a9d75fdae52e5e157:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:15:40,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:15:40,040 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:15:40,041 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102442 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:15:40,041 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:15:40,041 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): d7b388d8fe831b9a9d75fdae52e5e157/A is initiating minor compaction (all files) 2024-11-21T11:15:40,041 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): d7b388d8fe831b9a9d75fdae52e5e157/B is initiating minor compaction (all files) 2024-11-21T11:15:40,041 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7b388d8fe831b9a9d75fdae52e5e157/A in TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:40,042 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7b388d8fe831b9a9d75fdae52e5e157/B in TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:40,042 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/b33f350a8c0b4eb69baf00c4cecd9319, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/8f80b3425ce3490d86282ef746572ad7, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/35be6f145bfc45b58ec8b4dc6ab41fcd] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp, totalSize=100.0 K 2024-11-21T11:15:40,042 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/81ac939b1ff140529ff244b4a3d987eb, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/c469dd98db9446cbb676e15a962f2511, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/d69cabe4b8f4411f831e226acb7a005a] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp, totalSize=36.0 K 2024-11-21T11:15:40,042 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:40,042 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. files: [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/b33f350a8c0b4eb69baf00c4cecd9319, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/8f80b3425ce3490d86282ef746572ad7, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/35be6f145bfc45b58ec8b4dc6ab41fcd] 2024-11-21T11:15:40,042 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 81ac939b1ff140529ff244b4a3d987eb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732187732680 2024-11-21T11:15:40,042 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting b33f350a8c0b4eb69baf00c4cecd9319, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732187732680 2024-11-21T11:15:40,042 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f80b3425ce3490d86282ef746572ad7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732187735068 2024-11-21T11:15:40,043 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting c469dd98db9446cbb676e15a962f2511, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732187735068 2024-11-21T11:15:40,043 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35be6f145bfc45b58ec8b4dc6ab41fcd, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732187736176 2024-11-21T11:15:40,043 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting d69cabe4b8f4411f831e226acb7a005a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732187736176 2024-11-21T11:15:40,049 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:40,052 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7b388d8fe831b9a9d75fdae52e5e157#B#compaction#351 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:15:40,053 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/b94d1ecf5d624941a50a4ad5eb82a9bc is 50, key is test_row_0/B:col10/1732187736194/Put/seqid=0 2024-11-21T11:15:40,054 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121ade05ab3b69448d4a35ed851fe4e7660_d7b388d8fe831b9a9d75fdae52e5e157 store=[table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:40,056 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121ade05ab3b69448d4a35ed851fe4e7660_d7b388d8fe831b9a9d75fdae52e5e157, store=[table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:40,056 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ade05ab3b69448d4a35ed851fe4e7660_d7b388d8fe831b9a9d75fdae52e5e157 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:40,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742244_1420 (size=12254) 2024-11-21T11:15:40,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742245_1421 (size=4469) 2024-11-21T11:15:40,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-21T11:15:40,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:40,113 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing d7b388d8fe831b9a9d75fdae52e5e157 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-21T11:15:40,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=A 2024-11-21T11:15:40,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:40,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=B 2024-11-21T11:15:40,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:40,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=C 2024-11-21T11:15:40,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:40,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121451ba96b028442a6b94c1247a0bdd556_d7b388d8fe831b9a9d75fdae52e5e157 is 50, key is test_row_0/A:col10/1732187738455/Put/seqid=0 2024-11-21T11:15:40,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742246_1422 (size=12454) 2024-11-21T11:15:40,463 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/b94d1ecf5d624941a50a4ad5eb82a9bc as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/b94d1ecf5d624941a50a4ad5eb82a9bc 2024-11-21T11:15:40,467 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7b388d8fe831b9a9d75fdae52e5e157#A#compaction#350 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:15:40,468 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/187d111af4fa4e6fadf875f7b3e3dfe2 is 175, key is test_row_0/A:col10/1732187736194/Put/seqid=0 2024-11-21T11:15:40,470 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7b388d8fe831b9a9d75fdae52e5e157/B of d7b388d8fe831b9a9d75fdae52e5e157 into b94d1ecf5d624941a50a4ad5eb82a9bc(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:40,470 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7b388d8fe831b9a9d75fdae52e5e157: 2024-11-21T11:15:40,470 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., storeName=d7b388d8fe831b9a9d75fdae52e5e157/B, priority=13, startTime=1732187740040; duration=0sec 2024-11-21T11:15:40,470 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:15:40,471 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7b388d8fe831b9a9d75fdae52e5e157:B 2024-11-21T11:15:40,471 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:15:40,471 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:15:40,472 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): d7b388d8fe831b9a9d75fdae52e5e157/C is initiating minor compaction (all files) 2024-11-21T11:15:40,472 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7b388d8fe831b9a9d75fdae52e5e157/C in TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:40,472 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/2ebaf9635f744700b05fcd6ea0c54040, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/1bbee339d520404a9c8cd20628ced9da, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/46f25cb9b233448fa9c9c353fb01de10] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp, totalSize=36.0 K 2024-11-21T11:15:40,472 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ebaf9635f744700b05fcd6ea0c54040, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732187732680 2024-11-21T11:15:40,473 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bbee339d520404a9c8cd20628ced9da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732187735068 2024-11-21T11:15:40,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742247_1423 (size=31208) 2024-11-21T11:15:40,473 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 46f25cb9b233448fa9c9c353fb01de10, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732187736176 2024-11-21T11:15:40,478 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/187d111af4fa4e6fadf875f7b3e3dfe2 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/187d111af4fa4e6fadf875f7b3e3dfe2 2024-11-21T11:15:40,481 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7b388d8fe831b9a9d75fdae52e5e157#C#compaction#353 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:15:40,482 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/ad9ad3b5c57a4739a97223f082cc6743 is 50, key is test_row_0/C:col10/1732187736194/Put/seqid=0 2024-11-21T11:15:40,484 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7b388d8fe831b9a9d75fdae52e5e157/A of d7b388d8fe831b9a9d75fdae52e5e157 into 187d111af4fa4e6fadf875f7b3e3dfe2(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:40,484 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7b388d8fe831b9a9d75fdae52e5e157: 2024-11-21T11:15:40,484 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., storeName=d7b388d8fe831b9a9d75fdae52e5e157/A, priority=13, startTime=1732187740040; duration=0sec 2024-11-21T11:15:40,484 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:40,484 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7b388d8fe831b9a9d75fdae52e5e157:A 2024-11-21T11:15:40,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T11:15:40,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742248_1424 (size=12254) 2024-11-21T11:15:40,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:40,529 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121451ba96b028442a6b94c1247a0bdd556_d7b388d8fe831b9a9d75fdae52e5e157 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121451ba96b028442a6b94c1247a0bdd556_d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:40,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/5478114e716245ad92dfa03713631de5, store: [table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:40,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/5478114e716245ad92dfa03713631de5 is 175, key is test_row_0/A:col10/1732187738455/Put/seqid=0 2024-11-21T11:15:40,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742249_1425 (size=31255) 2024-11-21T11:15:40,535 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=275, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/5478114e716245ad92dfa03713631de5 2024-11-21T11:15:40,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/d8676181450049a6b63ae781830125dd is 50, key is test_row_0/B:col10/1732187738455/Put/seqid=0 2024-11-21T11:15:40,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742250_1426 (size=12301) 2024-11-21T11:15:40,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:40,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:40,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:40,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187800691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:40,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:40,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:40,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:40,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:40,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187800691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:40,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:40,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:40,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:40,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:40,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187800795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:40,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:40,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:40,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:40,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187800795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:40,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:40,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:40,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:40,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:40,898 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/ad9ad3b5c57a4739a97223f082cc6743 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/ad9ad3b5c57a4739a97223f082cc6743 2024-11-21T11:15:40,903 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7b388d8fe831b9a9d75fdae52e5e157/C of d7b388d8fe831b9a9d75fdae52e5e157 into ad9ad3b5c57a4739a97223f082cc6743(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:40,903 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7b388d8fe831b9a9d75fdae52e5e157: 2024-11-21T11:15:40,903 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., storeName=d7b388d8fe831b9a9d75fdae52e5e157/C, priority=13, startTime=1732187740040; duration=0sec 2024-11-21T11:15:40,903 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:40,903 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7b388d8fe831b9a9d75fdae52e5e157:C 2024-11-21T11:15:40,947 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/d8676181450049a6b63ae781830125dd 2024-11-21T11:15:40,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/e8c58de66c5b4dcba812846beb683998 is 50, key is test_row_0/C:col10/1732187738455/Put/seqid=0 2024-11-21T11:15:40,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742251_1427 (size=12301) 2024-11-21T11:15:40,957 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/e8c58de66c5b4dcba812846beb683998 2024-11-21T11:15:40,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/5478114e716245ad92dfa03713631de5 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/5478114e716245ad92dfa03713631de5 2024-11-21T11:15:40,973 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/5478114e716245ad92dfa03713631de5, entries=150, sequenceid=275, filesize=30.5 K 2024-11-21T11:15:40,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/d8676181450049a6b63ae781830125dd as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/d8676181450049a6b63ae781830125dd 2024-11-21T11:15:40,978 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/d8676181450049a6b63ae781830125dd, entries=150, sequenceid=275, filesize=12.0 K 2024-11-21T11:15:40,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/e8c58de66c5b4dcba812846beb683998 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/e8c58de66c5b4dcba812846beb683998 2024-11-21T11:15:40,986 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/e8c58de66c5b4dcba812846beb683998, entries=150, sequenceid=275, filesize=12.0 K 2024-11-21T11:15:40,987 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for d7b388d8fe831b9a9d75fdae52e5e157 in 873ms, sequenceid=275, compaction requested=false 2024-11-21T11:15:40,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for d7b388d8fe831b9a9d75fdae52e5e157: 2024-11-21T11:15:40,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:40,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-21T11:15:40,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-21T11:15:40,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-21T11:15:40,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6400 sec 2024-11-21T11:15:40,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.6460 sec 2024-11-21T11:15:41,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:41,009 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7b388d8fe831b9a9d75fdae52e5e157 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-21T11:15:41,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=A 2024-11-21T11:15:41,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:41,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=B 2024-11-21T11:15:41,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:41,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=C 2024-11-21T11:15:41,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:41,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d89395f48c5e4604937b38546952af3a_d7b388d8fe831b9a9d75fdae52e5e157 is 50, key is test_row_0/A:col10/1732187740638/Put/seqid=0 2024-11-21T11:15:41,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742252_1428 (size=14994) 2024-11-21T11:15:41,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:41,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187801108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:41,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:41,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:41,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:41,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187801108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:41,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:41,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:41,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:41,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:41,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:41,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187801216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:41,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:41,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:41,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:41,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:41,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187801216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:41,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:41,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:41,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:41,424 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:41,427 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d89395f48c5e4604937b38546952af3a_d7b388d8fe831b9a9d75fdae52e5e157 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d89395f48c5e4604937b38546952af3a_d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:41,428 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/406881e2b76c4bb6be22e6797b2435da, store: [table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:41,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/406881e2b76c4bb6be22e6797b2435da is 175, key is test_row_0/A:col10/1732187740638/Put/seqid=0 2024-11-21T11:15:41,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742253_1429 (size=39949) 2024-11-21T11:15:41,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:41,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187801435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:41,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:41,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187801436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:41,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:41,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:41,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:41,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:41,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:41,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:41,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T11:15:41,495 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:15:41,496 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:15:41,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-21T11:15:41,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T11:15:41,497 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:15:41,498 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:15:41,498 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:15:41,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T11:15:41,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-21T11:15:41,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:41,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:41,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:41,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:41,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:41,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:41,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:41,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187801745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:41,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:41,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187801745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:41,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:41,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:41,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:41,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:41,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:41,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:41,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-21T11:15:41,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:41,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:41,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:41,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:41,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:41,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:41,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T11:15:41,833 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/406881e2b76c4bb6be22e6797b2435da 2024-11-21T11:15:41,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/02f4bb3828674073b19db974465039da is 50, key is test_row_0/B:col10/1732187740638/Put/seqid=0 2024-11-21T11:15:41,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742254_1430 (size=12301) 2024-11-21T11:15:41,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-21T11:15:41,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:41,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:41,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:41,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:41,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:41,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:42,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-21T11:15:42,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:42,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:42,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:42,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:42,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:42,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:42,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T11:15:42,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/02f4bb3828674073b19db974465039da 2024-11-21T11:15:42,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/a1fdb4d14f7c430caaed7b00ab8bb3cd is 50, key is test_row_0/C:col10/1732187740638/Put/seqid=0 2024-11-21T11:15:42,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742255_1431 (size=12301) 2024-11-21T11:15:42,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-21T11:15:42,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:42,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:42,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:42,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:42,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:42,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:42,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:42,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187802267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:42,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:42,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:42,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:42,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:42,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187802267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:42,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:42,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:42,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:42,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-21T11:15:42,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:42,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:42,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:42,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:42,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:42,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:42,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-21T11:15:42,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:42,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:42,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:42,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:42,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:42,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:42,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T11:15:42,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/a1fdb4d14f7c430caaed7b00ab8bb3cd 2024-11-21T11:15:42,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/406881e2b76c4bb6be22e6797b2435da as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/406881e2b76c4bb6be22e6797b2435da 2024-11-21T11:15:42,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/406881e2b76c4bb6be22e6797b2435da, entries=200, sequenceid=291, filesize=39.0 K 2024-11-21T11:15:42,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/02f4bb3828674073b19db974465039da as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/02f4bb3828674073b19db974465039da 2024-11-21T11:15:42,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/02f4bb3828674073b19db974465039da, entries=150, sequenceid=291, filesize=12.0 K 2024-11-21T11:15:42,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/a1fdb4d14f7c430caaed7b00ab8bb3cd as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/a1fdb4d14f7c430caaed7b00ab8bb3cd 2024-11-21T11:15:42,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/a1fdb4d14f7c430caaed7b00ab8bb3cd, entries=150, sequenceid=291, filesize=12.0 K 2024-11-21T11:15:42,675 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d7b388d8fe831b9a9d75fdae52e5e157 in 1666ms, sequenceid=291, compaction requested=true 2024-11-21T11:15:42,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7b388d8fe831b9a9d75fdae52e5e157: 2024-11-21T11:15:42,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7b388d8fe831b9a9d75fdae52e5e157:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:15:42,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:42,676 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:15:42,676 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:15:42,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7b388d8fe831b9a9d75fdae52e5e157:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:15:42,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:42,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7b388d8fe831b9a9d75fdae52e5e157:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:15:42,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:15:42,677 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102412 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:15:42,677 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): d7b388d8fe831b9a9d75fdae52e5e157/A is initiating minor compaction (all files) 2024-11-21T11:15:42,677 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36856 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:15:42,677 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7b388d8fe831b9a9d75fdae52e5e157/A in TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:42,677 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): d7b388d8fe831b9a9d75fdae52e5e157/B is initiating minor compaction (all files) 2024-11-21T11:15:42,677 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7b388d8fe831b9a9d75fdae52e5e157/B in TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:42,677 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/187d111af4fa4e6fadf875f7b3e3dfe2, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/5478114e716245ad92dfa03713631de5, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/406881e2b76c4bb6be22e6797b2435da] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp, totalSize=100.0 K 2024-11-21T11:15:42,677 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:42,677 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/b94d1ecf5d624941a50a4ad5eb82a9bc, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/d8676181450049a6b63ae781830125dd, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/02f4bb3828674073b19db974465039da] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp, totalSize=36.0 K 2024-11-21T11:15:42,677 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. files: [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/187d111af4fa4e6fadf875f7b3e3dfe2, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/5478114e716245ad92dfa03713631de5, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/406881e2b76c4bb6be22e6797b2435da] 2024-11-21T11:15:42,678 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting b94d1ecf5d624941a50a4ad5eb82a9bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732187736176 2024-11-21T11:15:42,678 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 187d111af4fa4e6fadf875f7b3e3dfe2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732187736176 2024-11-21T11:15:42,679 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting d8676181450049a6b63ae781830125dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732187738424 2024-11-21T11:15:42,679 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 02f4bb3828674073b19db974465039da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732187740638 2024-11-21T11:15:42,680 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5478114e716245ad92dfa03713631de5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732187738424 2024-11-21T11:15:42,681 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 406881e2b76c4bb6be22e6797b2435da, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732187740638 2024-11-21T11:15:42,688 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7b388d8fe831b9a9d75fdae52e5e157#B#compaction#359 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:15:42,689 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/412d50cdda9f453eba6655291c7aa823 is 50, key is test_row_0/B:col10/1732187740638/Put/seqid=0 2024-11-21T11:15:42,704 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:42,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742256_1432 (size=12507) 2024-11-21T11:15:42,709 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121dadd3a150422431ca85fb8bff5b919ca_d7b388d8fe831b9a9d75fdae52e5e157 store=[table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:42,711 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121dadd3a150422431ca85fb8bff5b919ca_d7b388d8fe831b9a9d75fdae52e5e157, store=[table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:42,712 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121dadd3a150422431ca85fb8bff5b919ca_d7b388d8fe831b9a9d75fdae52e5e157 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:42,712 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/412d50cdda9f453eba6655291c7aa823 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/412d50cdda9f453eba6655291c7aa823 2024-11-21T11:15:42,717 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7b388d8fe831b9a9d75fdae52e5e157/B of d7b388d8fe831b9a9d75fdae52e5e157 into 412d50cdda9f453eba6655291c7aa823(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:42,717 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7b388d8fe831b9a9d75fdae52e5e157: 2024-11-21T11:15:42,718 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., storeName=d7b388d8fe831b9a9d75fdae52e5e157/B, priority=13, startTime=1732187742676; duration=0sec 2024-11-21T11:15:42,718 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:15:42,718 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7b388d8fe831b9a9d75fdae52e5e157:B 2024-11-21T11:15:42,718 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:15:42,719 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36856 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:15:42,719 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): d7b388d8fe831b9a9d75fdae52e5e157/C is initiating minor compaction (all files) 2024-11-21T11:15:42,719 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7b388d8fe831b9a9d75fdae52e5e157/C in TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:42,719 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/ad9ad3b5c57a4739a97223f082cc6743, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/e8c58de66c5b4dcba812846beb683998, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/a1fdb4d14f7c430caaed7b00ab8bb3cd] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp, totalSize=36.0 K 2024-11-21T11:15:42,720 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting ad9ad3b5c57a4739a97223f082cc6743, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732187736176 2024-11-21T11:15:42,720 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting e8c58de66c5b4dcba812846beb683998, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732187738424 2024-11-21T11:15:42,720 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting a1fdb4d14f7c430caaed7b00ab8bb3cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732187740638 2024-11-21T11:15:42,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742257_1433 (size=4469) 2024-11-21T11:15:42,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-21T11:15:42,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:42,729 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing d7b388d8fe831b9a9d75fdae52e5e157 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T11:15:42,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=A 2024-11-21T11:15:42,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:42,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=B 2024-11-21T11:15:42,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:42,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=C 2024-11-21T11:15:42,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:42,730 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7b388d8fe831b9a9d75fdae52e5e157#C#compaction#361 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:15:42,730 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/c2697b8984d846258601abe1b6c4587b is 50, key is test_row_0/C:col10/1732187740638/Put/seqid=0 2024-11-21T11:15:42,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742258_1434 (size=12507) 2024-11-21T11:15:42,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216214ba5beaf5447c8d4229fb000f079d_d7b388d8fe831b9a9d75fdae52e5e157 is 50, key is test_row_0/A:col10/1732187741089/Put/seqid=0 2024-11-21T11:15:42,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742259_1435 (size=12454) 2024-11-21T11:15:43,126 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7b388d8fe831b9a9d75fdae52e5e157#A#compaction#360 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:15:43,127 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/66512ed368ce431984a285f41364c735 is 175, key is test_row_0/A:col10/1732187740638/Put/seqid=0 2024-11-21T11:15:43,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742260_1436 (size=31461) 2024-11-21T11:15:43,140 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/c2697b8984d846258601abe1b6c4587b as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/c2697b8984d846258601abe1b6c4587b 2024-11-21T11:15:43,145 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7b388d8fe831b9a9d75fdae52e5e157/C of d7b388d8fe831b9a9d75fdae52e5e157 into c2697b8984d846258601abe1b6c4587b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:43,145 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7b388d8fe831b9a9d75fdae52e5e157: 2024-11-21T11:15:43,145 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., storeName=d7b388d8fe831b9a9d75fdae52e5e157/C, priority=13, startTime=1732187742676; duration=0sec 2024-11-21T11:15:43,145 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:43,145 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7b388d8fe831b9a9d75fdae52e5e157:C 2024-11-21T11:15:43,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:43,154 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216214ba5beaf5447c8d4229fb000f079d_d7b388d8fe831b9a9d75fdae52e5e157 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216214ba5beaf5447c8d4229fb000f079d_d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:43,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/0519a5471d74427d8d0ea303b5bb1707, store: [table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:43,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/0519a5471d74427d8d0ea303b5bb1707 is 175, key is test_row_0/A:col10/1732187741089/Put/seqid=0 2024-11-21T11:15:43,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742261_1437 (size=31255) 2024-11-21T11:15:43,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:43,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. as already flushing 2024-11-21T11:15:43,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:43,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35334 deadline: 1732187803325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:43,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:43,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187803326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:43,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:43,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:43,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:43,332 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_2' in d7b388d8fe831b9a9d75fdae52e5e157 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8262 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:15:43,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:43,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:43,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:43,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:43,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187803329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:43,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:43,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:43,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:43,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:43,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35324 deadline: 1732187803335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:43,347 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_0' in d7b388d8fe831b9a9d75fdae52e5e157 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8273 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:15:43,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:43,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:43,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:43,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:43,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1732187803346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:43,359 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_0' in d7b388d8fe831b9a9d75fdae52e5e157 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8286 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:15:43,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:43,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:43,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:43,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:43,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187803435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:43,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:43,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:43,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:43,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:43,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187803456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:43,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:43,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:43,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:43,536 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/66512ed368ce431984a285f41364c735 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/66512ed368ce431984a285f41364c735 2024-11-21T11:15:43,540 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7b388d8fe831b9a9d75fdae52e5e157/A of d7b388d8fe831b9a9d75fdae52e5e157 into 66512ed368ce431984a285f41364c735(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:43,540 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7b388d8fe831b9a9d75fdae52e5e157: 2024-11-21T11:15:43,540 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., storeName=d7b388d8fe831b9a9d75fdae52e5e157/A, priority=13, startTime=1732187742676; duration=0sec 2024-11-21T11:15:43,540 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:43,540 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7b388d8fe831b9a9d75fdae52e5e157:A 2024-11-21T11:15:43,560 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=311, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/0519a5471d74427d8d0ea303b5bb1707 2024-11-21T11:15:43,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/a3b6e627d4704931999f1c109a353467 is 50, key is test_row_0/B:col10/1732187741089/Put/seqid=0 2024-11-21T11:15:43,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742262_1438 (size=12301) 2024-11-21T11:15:43,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T11:15:43,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:43,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187803655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:43,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:43,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:43,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:43,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:43,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187803666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:43,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:43,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:43,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:43,978 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/a3b6e627d4704931999f1c109a353467 2024-11-21T11:15:43,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:43,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187803975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:43,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:43,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:43,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:43,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/a62b2a81a45f4b6596c42e6bb18e2560 is 50, key is test_row_0/C:col10/1732187741089/Put/seqid=0 2024-11-21T11:15:43,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742263_1439 (size=12301) 2024-11-21T11:15:43,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:43,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187803985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:43,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:43,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:43,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:44,391 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/a62b2a81a45f4b6596c42e6bb18e2560 2024-11-21T11:15:44,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/0519a5471d74427d8d0ea303b5bb1707 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/0519a5471d74427d8d0ea303b5bb1707 2024-11-21T11:15:44,399 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/0519a5471d74427d8d0ea303b5bb1707, entries=150, sequenceid=311, filesize=30.5 K 2024-11-21T11:15:44,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/a3b6e627d4704931999f1c109a353467 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/a3b6e627d4704931999f1c109a353467 2024-11-21T11:15:44,403 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/a3b6e627d4704931999f1c109a353467, entries=150, sequenceid=311, filesize=12.0 K 2024-11-21T11:15:44,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/a62b2a81a45f4b6596c42e6bb18e2560 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/a62b2a81a45f4b6596c42e6bb18e2560 2024-11-21T11:15:44,408 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/a62b2a81a45f4b6596c42e6bb18e2560, entries=150, sequenceid=311, filesize=12.0 K 2024-11-21T11:15:44,408 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for d7b388d8fe831b9a9d75fdae52e5e157 in 1679ms, sequenceid=311, compaction requested=false 2024-11-21T11:15:44,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for d7b388d8fe831b9a9d75fdae52e5e157: 2024-11-21T11:15:44,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:44,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-21T11:15:44,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-21T11:15:44,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-21T11:15:44,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9120 sec 2024-11-21T11:15:44,413 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.9160 sec 2024-11-21T11:15:44,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:44,499 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7b388d8fe831b9a9d75fdae52e5e157 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T11:15:44,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=A 2024-11-21T11:15:44,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:44,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=B 2024-11-21T11:15:44,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:44,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=C 2024-11-21T11:15:44,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:44,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c1618333cb6f4e6baf57d0e63a16d472_d7b388d8fe831b9a9d75fdae52e5e157 is 50, key is test_row_0/A:col10/1732187743327/Put/seqid=0 2024-11-21T11:15:44,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742264_1440 (size=14994) 2024-11-21T11:15:44,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187804565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:44,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:44,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:44,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:44,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:44,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187804573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:44,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:44,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:44,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:44,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:44,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187804675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:44,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:44,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:44,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:44,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:44,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187804696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:44,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:44,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:44,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:44,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:44,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187804885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:44,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:44,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:44,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:44,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:44,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187804915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:44,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:44,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:44,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:44,919 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:44,922 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c1618333cb6f4e6baf57d0e63a16d472_d7b388d8fe831b9a9d75fdae52e5e157 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c1618333cb6f4e6baf57d0e63a16d472_d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:44,923 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/492029ee2a164223b335840b97c6d49c, store: [table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:44,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/492029ee2a164223b335840b97c6d49c is 175, key is test_row_0/A:col10/1732187743327/Put/seqid=0 2024-11-21T11:15:44,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742265_1441 (size=39949) 2024-11-21T11:15:44,927 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=327, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/492029ee2a164223b335840b97c6d49c 2024-11-21T11:15:44,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/85351c2617aa4038b7a78f143fab6ca3 is 50, key is test_row_0/B:col10/1732187743327/Put/seqid=0 2024-11-21T11:15:44,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742266_1442 (size=12301) 2024-11-21T11:15:45,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:45,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187805206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:45,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:45,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:45,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:45,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:45,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187805225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:45,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:45,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:45,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:45,339 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/85351c2617aa4038b7a78f143fab6ca3 2024-11-21T11:15:45,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/7b407220a3d24d6a82cb375f62b3b09e is 50, key is test_row_0/C:col10/1732187743327/Put/seqid=0 2024-11-21T11:15:45,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742267_1443 (size=12301) 2024-11-21T11:15:45,572 INFO [Thread-1659 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1659. 2024-11-21T11:15:45,572 DEBUG [Thread-1659 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:15:45,572 DEBUG [Thread-1659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:45,572 DEBUG [Thread-1659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:45,572 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:15:45,579 INFO [Thread-1663 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1663. 2024-11-21T11:15:45,579 DEBUG [Thread-1663 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:15:45,579 DEBUG [Thread-1663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:45,579 DEBUG [Thread-1663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:45,580 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:15:45,580 INFO [Thread-1657 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1657. 2024-11-21T11:15:45,580 DEBUG [Thread-1657 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:15:45,580 DEBUG [Thread-1657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:45,580 DEBUG [Thread-1657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:45,580 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:15:45,583 INFO [Thread-1661 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1661. 2024-11-21T11:15:45,583 DEBUG [Thread-1661 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:15:45,583 DEBUG [Thread-1661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:45,583 DEBUG [Thread-1661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:45,583 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:15:45,586 INFO [Thread-1665 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1665. 2024-11-21T11:15:45,586 DEBUG [Thread-1665 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:15:45,586 DEBUG [Thread-1665 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:45,587 DEBUG [Thread-1665 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:45,587 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:15:45,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T11:15:45,655 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:15:45,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:45,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1732187805726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:45,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:45,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:45,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:45,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:45,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35300 deadline: 1732187805745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:45,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:45,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:45,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:15:45,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/7b407220a3d24d6a82cb375f62b3b09e 2024-11-21T11:15:45,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/492029ee2a164223b335840b97c6d49c as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/492029ee2a164223b335840b97c6d49c 2024-11-21T11:15:45,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/492029ee2a164223b335840b97c6d49c, entries=200, sequenceid=327, filesize=39.0 K 2024-11-21T11:15:45,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/85351c2617aa4038b7a78f143fab6ca3 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/85351c2617aa4038b7a78f143fab6ca3 2024-11-21T11:15:45,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/85351c2617aa4038b7a78f143fab6ca3, entries=150, sequenceid=327, filesize=12.0 K 2024-11-21T11:15:45,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/7b407220a3d24d6a82cb375f62b3b09e as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/7b407220a3d24d6a82cb375f62b3b09e 2024-11-21T11:15:45,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/7b407220a3d24d6a82cb375f62b3b09e, entries=150, sequenceid=327, filesize=12.0 K 2024-11-21T11:15:45,765 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for d7b388d8fe831b9a9d75fdae52e5e157 in 1266ms, sequenceid=327, compaction requested=true 2024-11-21T11:15:45,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7b388d8fe831b9a9d75fdae52e5e157: 2024-11-21T11:15:45,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7b388d8fe831b9a9d75fdae52e5e157:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:15:45,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:45,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7b388d8fe831b9a9d75fdae52e5e157:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:15:45,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:45,766 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:15:45,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7b388d8fe831b9a9d75fdae52e5e157:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:15:45,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:15:45,766 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:15:45,766 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102665 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:15:45,766 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37109 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:15:45,766 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): d7b388d8fe831b9a9d75fdae52e5e157/A is initiating minor compaction (all files) 2024-11-21T11:15:45,767 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): d7b388d8fe831b9a9d75fdae52e5e157/B is initiating minor compaction (all files) 2024-11-21T11:15:45,767 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7b388d8fe831b9a9d75fdae52e5e157/A in TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:45,767 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7b388d8fe831b9a9d75fdae52e5e157/B in TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:45,767 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/66512ed368ce431984a285f41364c735, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/0519a5471d74427d8d0ea303b5bb1707, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/492029ee2a164223b335840b97c6d49c] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp, totalSize=100.3 K 2024-11-21T11:15:45,767 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/412d50cdda9f453eba6655291c7aa823, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/a3b6e627d4704931999f1c109a353467, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/85351c2617aa4038b7a78f143fab6ca3] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp, totalSize=36.2 K 2024-11-21T11:15:45,767 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:45,767 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. files: [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/66512ed368ce431984a285f41364c735, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/0519a5471d74427d8d0ea303b5bb1707, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/492029ee2a164223b335840b97c6d49c] 2024-11-21T11:15:45,767 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 412d50cdda9f453eba6655291c7aa823, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732187740638 2024-11-21T11:15:45,767 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66512ed368ce431984a285f41364c735, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732187740638 2024-11-21T11:15:45,767 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting a3b6e627d4704931999f1c109a353467, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732187741089 2024-11-21T11:15:45,767 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0519a5471d74427d8d0ea303b5bb1707, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732187741089 2024-11-21T11:15:45,767 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 85351c2617aa4038b7a78f143fab6ca3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732187743317 2024-11-21T11:15:45,767 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 492029ee2a164223b335840b97c6d49c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732187743317 2024-11-21T11:15:45,773 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:45,774 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112132a7a414cd2d4c0a9741f5fc16432a51_d7b388d8fe831b9a9d75fdae52e5e157 store=[table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:45,774 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7b388d8fe831b9a9d75fdae52e5e157#B#compaction#369 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:15:45,775 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/05bec5b4fb1047d299f850661657edd7 is 50, key is test_row_0/B:col10/1732187743327/Put/seqid=0 2024-11-21T11:15:45,779 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112132a7a414cd2d4c0a9741f5fc16432a51_d7b388d8fe831b9a9d75fdae52e5e157, store=[table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:45,779 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112132a7a414cd2d4c0a9741f5fc16432a51_d7b388d8fe831b9a9d75fdae52e5e157 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:45,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742268_1444 (size=12609) 2024-11-21T11:15:45,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742269_1445 (size=4469) 2024-11-21T11:15:45,791 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7b388d8fe831b9a9d75fdae52e5e157#A#compaction#368 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:15:45,791 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/2971357769a14a988f197baba4e2afe2 is 175, key is test_row_0/A:col10/1732187743327/Put/seqid=0 2024-11-21T11:15:45,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742270_1446 (size=31563) 2024-11-21T11:15:46,190 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/05bec5b4fb1047d299f850661657edd7 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/05bec5b4fb1047d299f850661657edd7 2024-11-21T11:15:46,194 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7b388d8fe831b9a9d75fdae52e5e157/B of d7b388d8fe831b9a9d75fdae52e5e157 into 05bec5b4fb1047d299f850661657edd7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:46,194 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7b388d8fe831b9a9d75fdae52e5e157: 2024-11-21T11:15:46,194 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., storeName=d7b388d8fe831b9a9d75fdae52e5e157/B, priority=13, startTime=1732187745766; duration=0sec 2024-11-21T11:15:46,194 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:15:46,194 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7b388d8fe831b9a9d75fdae52e5e157:B 2024-11-21T11:15:46,194 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:15:46,195 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37109 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:15:46,195 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): d7b388d8fe831b9a9d75fdae52e5e157/C is initiating minor compaction (all files) 2024-11-21T11:15:46,195 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7b388d8fe831b9a9d75fdae52e5e157/C in TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:46,195 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/c2697b8984d846258601abe1b6c4587b, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/a62b2a81a45f4b6596c42e6bb18e2560, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/7b407220a3d24d6a82cb375f62b3b09e] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp, totalSize=36.2 K 2024-11-21T11:15:46,195 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting c2697b8984d846258601abe1b6c4587b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732187740638 2024-11-21T11:15:46,195 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting a62b2a81a45f4b6596c42e6bb18e2560, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732187741089 2024-11-21T11:15:46,195 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b407220a3d24d6a82cb375f62b3b09e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732187743317 2024-11-21T11:15:46,198 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/2971357769a14a988f197baba4e2afe2 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/2971357769a14a988f197baba4e2afe2 2024-11-21T11:15:46,202 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7b388d8fe831b9a9d75fdae52e5e157#C#compaction#370 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:15:46,202 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7b388d8fe831b9a9d75fdae52e5e157/A of d7b388d8fe831b9a9d75fdae52e5e157 into 2971357769a14a988f197baba4e2afe2(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:46,202 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7b388d8fe831b9a9d75fdae52e5e157: 2024-11-21T11:15:46,202 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., storeName=d7b388d8fe831b9a9d75fdae52e5e157/A, priority=13, startTime=1732187745766; duration=0sec 2024-11-21T11:15:46,202 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:46,202 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/9c3eaa3759494cf8974360caefa44483 is 50, key is test_row_0/C:col10/1732187743327/Put/seqid=0 2024-11-21T11:15:46,202 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7b388d8fe831b9a9d75fdae52e5e157:A 2024-11-21T11:15:46,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742271_1447 (size=12609) 2024-11-21T11:15:46,610 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/9c3eaa3759494cf8974360caefa44483 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/9c3eaa3759494cf8974360caefa44483 2024-11-21T11:15:46,613 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7b388d8fe831b9a9d75fdae52e5e157/C of d7b388d8fe831b9a9d75fdae52e5e157 into 9c3eaa3759494cf8974360caefa44483(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:15:46,614 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7b388d8fe831b9a9d75fdae52e5e157: 2024-11-21T11:15:46,614 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., storeName=d7b388d8fe831b9a9d75fdae52e5e157/C, priority=13, startTime=1732187745766; duration=0sec 2024-11-21T11:15:46,614 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:15:46,614 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7b388d8fe831b9a9d75fdae52e5e157:C 2024-11-21T11:15:46,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:46,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7b388d8fe831b9a9d75fdae52e5e157 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T11:15:46,737 INFO [Thread-1648 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1648. 2024-11-21T11:15:46,737 DEBUG [Thread-1648 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:15:46,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=A 2024-11-21T11:15:46,737 DEBUG [Thread-1648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:46,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:46,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=B 2024-11-21T11:15:46,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:46,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=C 2024-11-21T11:15:46,737 DEBUG [Thread-1648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:46,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:46,737 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:15:46,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b5352e45f37e4047977cd21addd9efd0_d7b388d8fe831b9a9d75fdae52e5e157 is 50, key is test_row_0/A:col10/1732187744506/Put/seqid=0 2024-11-21T11:15:46,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742272_1448 (size=12454) 2024-11-21T11:15:46,766 INFO [Thread-1654 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1654. 2024-11-21T11:15:46,766 DEBUG [Thread-1654 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:15:46,766 DEBUG [Thread-1654 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:46,766 DEBUG [Thread-1654 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:46,766 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:15:47,146 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:47,149 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b5352e45f37e4047977cd21addd9efd0_d7b388d8fe831b9a9d75fdae52e5e157 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b5352e45f37e4047977cd21addd9efd0_d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:47,150 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/ad8d24b264c447e782315b0d83d9468a, store: [table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:47,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/ad8d24b264c447e782315b0d83d9468a is 175, key is test_row_0/A:col10/1732187744506/Put/seqid=0 2024-11-21T11:15:47,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742273_1449 (size=31255) 2024-11-21T11:15:47,555 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=349, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/ad8d24b264c447e782315b0d83d9468a 2024-11-21T11:15:47,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/d5956456dcd44fa890b093c90d33930a is 50, key is test_row_0/B:col10/1732187744506/Put/seqid=0 2024-11-21T11:15:47,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742274_1450 (size=12301) 2024-11-21T11:15:47,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/d5956456dcd44fa890b093c90d33930a 2024-11-21T11:15:47,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/41ffcc51b87b4fc0988754e65d3ac4be is 50, key is test_row_0/C:col10/1732187744506/Put/seqid=0 2024-11-21T11:15:47,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742275_1451 (size=12301) 2024-11-21T11:15:48,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/41ffcc51b87b4fc0988754e65d3ac4be 2024-11-21T11:15:48,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/ad8d24b264c447e782315b0d83d9468a as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/ad8d24b264c447e782315b0d83d9468a 2024-11-21T11:15:48,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/ad8d24b264c447e782315b0d83d9468a, entries=150, sequenceid=349, filesize=30.5 K 2024-11-21T11:15:48,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/d5956456dcd44fa890b093c90d33930a as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/d5956456dcd44fa890b093c90d33930a 2024-11-21T11:15:48,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/d5956456dcd44fa890b093c90d33930a, entries=150, sequenceid=349, filesize=12.0 K 2024-11-21T11:15:48,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/41ffcc51b87b4fc0988754e65d3ac4be as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/41ffcc51b87b4fc0988754e65d3ac4be 2024-11-21T11:15:48,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/41ffcc51b87b4fc0988754e65d3ac4be, entries=150, sequenceid=349, filesize=12.0 K 2024-11-21T11:15:48,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=6.71 KB/6870 for d7b388d8fe831b9a9d75fdae52e5e157 in 1650ms, sequenceid=349, compaction requested=false 2024-11-21T11:15:48,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7b388d8fe831b9a9d75fdae52e5e157: 2024-11-21T11:15:53,387 INFO [Thread-1650 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1650. 2024-11-21T11:15:53,387 DEBUG [Thread-1650 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:15:53,387 DEBUG [Thread-1650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:53,387 DEBUG [Thread-1650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:53,387 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:15:53,427 INFO [Thread-1646 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1646. 2024-11-21T11:15:53,427 DEBUG [Thread-1646 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:15:53,427 DEBUG [Thread-1646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:53,427 DEBUG [Thread-1646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:53,427 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:15:53,457 INFO [Thread-1652 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1652. 2024-11-21T11:15:53,457 DEBUG [Thread-1652 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:15:53,457 DEBUG [Thread-1652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:53,457 DEBUG [Thread-1652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:53,457 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:15:53,457 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T11:15:53,457 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-21T11:15:53,457 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-21T11:15:53,457 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-21T11:15:53,457 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-21T11:15:53,457 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-21T11:15:53,457 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T11:15:53,457 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T11:15:53,457 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1026 2024-11-21T11:15:53,457 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3073 rows 2024-11-21T11:15:53,457 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1028 2024-11-21T11:15:53,457 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3079 rows 2024-11-21T11:15:53,457 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1023 2024-11-21T11:15:53,457 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3065 rows 2024-11-21T11:15:53,457 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1030 2024-11-21T11:15:53,457 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3082 rows 2024-11-21T11:15:53,457 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1033 2024-11-21T11:15:53,457 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3094 rows 2024-11-21T11:15:53,457 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T11:15:53,457 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T11:15:53,457 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:53,457 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:53,457 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:15:53,458 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T11:15:53,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T11:15:53,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T11:15:53,461 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732187753461"}]},"ts":"1732187753461"} 2024-11-21T11:15:53,462 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T11:15:53,462 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T11:15:53,463 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T11:15:53,464 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7b388d8fe831b9a9d75fdae52e5e157, UNASSIGN}] 2024-11-21T11:15:53,464 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7b388d8fe831b9a9d75fdae52e5e157, UNASSIGN 2024-11-21T11:15:53,465 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=d7b388d8fe831b9a9d75fdae52e5e157, regionState=CLOSING, regionLocation=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:53,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7b388d8fe831b9a9d75fdae52e5e157, UNASSIGN because future has completed 2024-11-21T11:15:53,466 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T11:15:53,466 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276}] 2024-11-21T11:15:53,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T11:15:53,618 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(122): Close d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:53,618 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T11:15:53,618 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1722): Closing d7b388d8fe831b9a9d75fdae52e5e157, disabling compactions & flushes 2024-11-21T11:15:53,618 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:53,618 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:53,618 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. after waiting 0 ms 2024-11-21T11:15:53,618 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:53,618 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(2902): Flushing d7b388d8fe831b9a9d75fdae52e5e157 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-21T11:15:53,618 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=A 2024-11-21T11:15:53,619 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:53,619 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=B 2024-11-21T11:15:53,619 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:53,619 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7b388d8fe831b9a9d75fdae52e5e157, store=C 2024-11-21T11:15:53,619 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:53,623 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219257cfe14af0414aa47ec3e5a72aac55_d7b388d8fe831b9a9d75fdae52e5e157 is 50, key is test_row_0/A:col10/1732187753455/Put/seqid=0 2024-11-21T11:15:53,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742276_1452 (size=9914) 2024-11-21T11:15:53,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T11:15:54,027 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:54,031 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219257cfe14af0414aa47ec3e5a72aac55_d7b388d8fe831b9a9d75fdae52e5e157 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219257cfe14af0414aa47ec3e5a72aac55_d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:54,032 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/eca05a1dfbef445a94be1736222d387c, store: [table=TestAcidGuarantees family=A region=d7b388d8fe831b9a9d75fdae52e5e157] 2024-11-21T11:15:54,032 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/eca05a1dfbef445a94be1736222d387c is 175, key is test_row_0/A:col10/1732187753455/Put/seqid=0 2024-11-21T11:15:54,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742277_1453 (size=22561) 2024-11-21T11:15:54,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T11:15:54,436 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=356, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/eca05a1dfbef445a94be1736222d387c 2024-11-21T11:15:54,442 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/20a3fafeccb84c2d9799b3b8a8add7d4 is 50, key is test_row_0/B:col10/1732187753455/Put/seqid=0 2024-11-21T11:15:54,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742278_1454 (size=9857) 2024-11-21T11:15:54,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T11:15:54,846 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/20a3fafeccb84c2d9799b3b8a8add7d4 2024-11-21T11:15:54,852 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/e90f9369e14847a5b1fb83a55b8beff2 is 50, key is test_row_0/C:col10/1732187753455/Put/seqid=0 2024-11-21T11:15:54,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742279_1455 (size=9857) 2024-11-21T11:15:55,256 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/e90f9369e14847a5b1fb83a55b8beff2 2024-11-21T11:15:55,260 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/A/eca05a1dfbef445a94be1736222d387c as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/eca05a1dfbef445a94be1736222d387c 2024-11-21T11:15:55,263 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/eca05a1dfbef445a94be1736222d387c, entries=100, sequenceid=356, filesize=22.0 K 2024-11-21T11:15:55,264 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/B/20a3fafeccb84c2d9799b3b8a8add7d4 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/20a3fafeccb84c2d9799b3b8a8add7d4 2024-11-21T11:15:55,266 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/20a3fafeccb84c2d9799b3b8a8add7d4, entries=100, sequenceid=356, filesize=9.6 K 2024-11-21T11:15:55,267 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/.tmp/C/e90f9369e14847a5b1fb83a55b8beff2 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/e90f9369e14847a5b1fb83a55b8beff2 2024-11-21T11:15:55,270 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/e90f9369e14847a5b1fb83a55b8beff2, entries=100, sequenceid=356, filesize=9.6 K 2024-11-21T11:15:55,271 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for d7b388d8fe831b9a9d75fdae52e5e157 in 1653ms, sequenceid=356, compaction requested=true 2024-11-21T11:15:55,271 DEBUG [StoreCloser-TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/b33f350a8c0b4eb69baf00c4cecd9319, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/8f80b3425ce3490d86282ef746572ad7, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/35be6f145bfc45b58ec8b4dc6ab41fcd, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/187d111af4fa4e6fadf875f7b3e3dfe2, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/5478114e716245ad92dfa03713631de5, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/406881e2b76c4bb6be22e6797b2435da, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/66512ed368ce431984a285f41364c735, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/0519a5471d74427d8d0ea303b5bb1707, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/492029ee2a164223b335840b97c6d49c] to archive 2024-11-21T11:15:55,272 DEBUG [StoreCloser-TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T11:15:55,273 DEBUG [StoreCloser-TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/b33f350a8c0b4eb69baf00c4cecd9319 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/b33f350a8c0b4eb69baf00c4cecd9319 2024-11-21T11:15:55,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/8f80b3425ce3490d86282ef746572ad7 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/8f80b3425ce3490d86282ef746572ad7 2024-11-21T11:15:55,275 DEBUG [StoreCloser-TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/35be6f145bfc45b58ec8b4dc6ab41fcd to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/35be6f145bfc45b58ec8b4dc6ab41fcd 2024-11-21T11:15:55,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/187d111af4fa4e6fadf875f7b3e3dfe2 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/187d111af4fa4e6fadf875f7b3e3dfe2 2024-11-21T11:15:55,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/5478114e716245ad92dfa03713631de5 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/5478114e716245ad92dfa03713631de5 2024-11-21T11:15:55,277 DEBUG [StoreCloser-TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/406881e2b76c4bb6be22e6797b2435da to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/406881e2b76c4bb6be22e6797b2435da 2024-11-21T11:15:55,278 DEBUG [StoreCloser-TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/66512ed368ce431984a285f41364c735 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/66512ed368ce431984a285f41364c735 2024-11-21T11:15:55,279 DEBUG [StoreCloser-TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/0519a5471d74427d8d0ea303b5bb1707 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/0519a5471d74427d8d0ea303b5bb1707 2024-11-21T11:15:55,280 DEBUG [StoreCloser-TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/492029ee2a164223b335840b97c6d49c to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/492029ee2a164223b335840b97c6d49c 2024-11-21T11:15:55,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/81ac939b1ff140529ff244b4a3d987eb, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/c469dd98db9446cbb676e15a962f2511, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/b94d1ecf5d624941a50a4ad5eb82a9bc, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/d69cabe4b8f4411f831e226acb7a005a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/d8676181450049a6b63ae781830125dd, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/412d50cdda9f453eba6655291c7aa823, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/02f4bb3828674073b19db974465039da, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/a3b6e627d4704931999f1c109a353467, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/85351c2617aa4038b7a78f143fab6ca3] to archive 2024-11-21T11:15:55,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T11:15:55,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/81ac939b1ff140529ff244b4a3d987eb to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/81ac939b1ff140529ff244b4a3d987eb 2024-11-21T11:15:55,283 DEBUG [StoreCloser-TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/c469dd98db9446cbb676e15a962f2511 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/c469dd98db9446cbb676e15a962f2511 2024-11-21T11:15:55,284 DEBUG [StoreCloser-TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/b94d1ecf5d624941a50a4ad5eb82a9bc to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/b94d1ecf5d624941a50a4ad5eb82a9bc 2024-11-21T11:15:55,285 DEBUG [StoreCloser-TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/d69cabe4b8f4411f831e226acb7a005a to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/d69cabe4b8f4411f831e226acb7a005a 2024-11-21T11:15:55,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/d8676181450049a6b63ae781830125dd to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/d8676181450049a6b63ae781830125dd 2024-11-21T11:15:55,287 DEBUG [StoreCloser-TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/412d50cdda9f453eba6655291c7aa823 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/412d50cdda9f453eba6655291c7aa823 2024-11-21T11:15:55,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/02f4bb3828674073b19db974465039da to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/02f4bb3828674073b19db974465039da 2024-11-21T11:15:55,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/a3b6e627d4704931999f1c109a353467 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/a3b6e627d4704931999f1c109a353467 2024-11-21T11:15:55,289 DEBUG [StoreCloser-TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/85351c2617aa4038b7a78f143fab6ca3 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/85351c2617aa4038b7a78f143fab6ca3 2024-11-21T11:15:55,290 DEBUG [StoreCloser-TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/2ebaf9635f744700b05fcd6ea0c54040, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/1bbee339d520404a9c8cd20628ced9da, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/ad9ad3b5c57a4739a97223f082cc6743, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/46f25cb9b233448fa9c9c353fb01de10, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/e8c58de66c5b4dcba812846beb683998, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/c2697b8984d846258601abe1b6c4587b, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/a1fdb4d14f7c430caaed7b00ab8bb3cd, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/a62b2a81a45f4b6596c42e6bb18e2560, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/7b407220a3d24d6a82cb375f62b3b09e] to archive 2024-11-21T11:15:55,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T11:15:55,292 DEBUG [StoreCloser-TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/2ebaf9635f744700b05fcd6ea0c54040 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/2ebaf9635f744700b05fcd6ea0c54040 2024-11-21T11:15:55,293 DEBUG [StoreCloser-TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/1bbee339d520404a9c8cd20628ced9da to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/1bbee339d520404a9c8cd20628ced9da 2024-11-21T11:15:55,294 DEBUG [StoreCloser-TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/ad9ad3b5c57a4739a97223f082cc6743 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/ad9ad3b5c57a4739a97223f082cc6743 2024-11-21T11:15:55,294 DEBUG [StoreCloser-TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/46f25cb9b233448fa9c9c353fb01de10 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/46f25cb9b233448fa9c9c353fb01de10 2024-11-21T11:15:55,295 DEBUG [StoreCloser-TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/e8c58de66c5b4dcba812846beb683998 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/e8c58de66c5b4dcba812846beb683998 2024-11-21T11:15:55,296 DEBUG [StoreCloser-TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/c2697b8984d846258601abe1b6c4587b to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/c2697b8984d846258601abe1b6c4587b 2024-11-21T11:15:55,297 DEBUG [StoreCloser-TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/a1fdb4d14f7c430caaed7b00ab8bb3cd to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/a1fdb4d14f7c430caaed7b00ab8bb3cd 2024-11-21T11:15:55,298 DEBUG [StoreCloser-TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/a62b2a81a45f4b6596c42e6bb18e2560 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/a62b2a81a45f4b6596c42e6bb18e2560 2024-11-21T11:15:55,298 DEBUG [StoreCloser-TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/7b407220a3d24d6a82cb375f62b3b09e to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/7b407220a3d24d6a82cb375f62b3b09e 2024-11-21T11:15:55,302 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/recovered.edits/359.seqid, newMaxSeqId=359, maxSeqId=4 2024-11-21T11:15:55,302 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. 2024-11-21T11:15:55,302 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1676): Region close journal for d7b388d8fe831b9a9d75fdae52e5e157: Waiting for close lock at 1732187753618Running coprocessor pre-close hooks at 1732187753618Disabling compacts and flushes for region at 1732187753618Disabling writes for close at 1732187753618Obtaining lock to block concurrent updates at 1732187753618Preparing flush snapshotting stores in d7b388d8fe831b9a9d75fdae52e5e157 at 1732187753618Finished memstore snapshotting TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732187753619 (+1 ms)Flushing stores of TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157. at 1732187753619Flushing d7b388d8fe831b9a9d75fdae52e5e157/A: creating writer at 1732187753619Flushing mob file d7b388d8fe831b9a9d75fdae52e5e157/A: appending metadata at 1732187753623 (+4 ms)Flushing mob file d7b388d8fe831b9a9d75fdae52e5e157/A: closing flushed file at 1732187753623Flushing d7b388d8fe831b9a9d75fdae52e5e157/A: appending metadata at 1732187754032 (+409 ms)Flushing d7b388d8fe831b9a9d75fdae52e5e157/A: closing flushed file at 1732187754032Flushing d7b388d8fe831b9a9d75fdae52e5e157/B: creating writer at 1732187754440 (+408 ms)Flushing d7b388d8fe831b9a9d75fdae52e5e157/B: appending metadata at 1732187754442 (+2 ms)Flushing d7b388d8fe831b9a9d75fdae52e5e157/B: closing flushed file at 1732187754442Flushing d7b388d8fe831b9a9d75fdae52e5e157/C: creating writer at 1732187754849 (+407 ms)Flushing d7b388d8fe831b9a9d75fdae52e5e157/C: appending metadata at 1732187754852 (+3 ms)Flushing d7b388d8fe831b9a9d75fdae52e5e157/C: closing flushed file at 1732187754852Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f5b6b8e: reopening flushed file at 1732187755259 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2994ae77: reopening flushed file at 1732187755263 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44c03447: reopening flushed file at 1732187755267 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for d7b388d8fe831b9a9d75fdae52e5e157 in 1653ms, sequenceid=356, compaction requested=true at 1732187755271 (+4 ms)Writing region close event to WAL at 1732187755299 (+28 ms)Running coprocessor post-close hooks at 1732187755302 (+3 ms)Closed at 1732187755302 2024-11-21T11:15:55,304 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(157): Closed d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:55,304 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=d7b388d8fe831b9a9d75fdae52e5e157, regionState=CLOSED 2024-11-21T11:15:55,305 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 because future has completed 2024-11-21T11:15:55,307 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-21T11:15:55,307 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseRegionProcedure d7b388d8fe831b9a9d75fdae52e5e157, server=7b462513bfc2,33417,1732187618276 in 1.8400 sec 2024-11-21T11:15:55,308 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-21T11:15:55,308 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7b388d8fe831b9a9d75fdae52e5e157, UNASSIGN in 1.8430 sec 2024-11-21T11:15:55,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-21T11:15:55,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8470 sec 2024-11-21T11:15:55,311 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732187755311"}]},"ts":"1732187755311"} 2024-11-21T11:15:55,312 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T11:15:55,312 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T11:15:55,313 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8540 sec 2024-11-21T11:15:55,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T11:15:55,606 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:15:55,606 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T11:15:55,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T11:15:55,608 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T11:15:55,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-21T11:15:55,608 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=117, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T11:15:55,611 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:55,613 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A, FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B, FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C, FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/recovered.edits] 2024-11-21T11:15:55,614 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/2971357769a14a988f197baba4e2afe2 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/2971357769a14a988f197baba4e2afe2 2024-11-21T11:15:55,615 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/ad8d24b264c447e782315b0d83d9468a to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/ad8d24b264c447e782315b0d83d9468a 2024-11-21T11:15:55,616 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/eca05a1dfbef445a94be1736222d387c to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/A/eca05a1dfbef445a94be1736222d387c 2024-11-21T11:15:55,618 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/05bec5b4fb1047d299f850661657edd7 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/05bec5b4fb1047d299f850661657edd7 2024-11-21T11:15:55,619 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/20a3fafeccb84c2d9799b3b8a8add7d4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/20a3fafeccb84c2d9799b3b8a8add7d4 2024-11-21T11:15:55,620 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/d5956456dcd44fa890b093c90d33930a to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/B/d5956456dcd44fa890b093c90d33930a 2024-11-21T11:15:55,621 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/41ffcc51b87b4fc0988754e65d3ac4be to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/41ffcc51b87b4fc0988754e65d3ac4be 2024-11-21T11:15:55,622 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/9c3eaa3759494cf8974360caefa44483 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/9c3eaa3759494cf8974360caefa44483 2024-11-21T11:15:55,623 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/e90f9369e14847a5b1fb83a55b8beff2 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/C/e90f9369e14847a5b1fb83a55b8beff2 2024-11-21T11:15:55,625 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/recovered.edits/359.seqid to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157/recovered.edits/359.seqid 2024-11-21T11:15:55,626 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:55,626 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T11:15:55,626 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T11:15:55,627 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-21T11:15:55,629 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112101c1c51e85b649cabdd82255461ae816_d7b388d8fe831b9a9d75fdae52e5e157 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112101c1c51e85b649cabdd82255461ae816_d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:55,629 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411210a39bb8a15d54749832801387d5a8599_d7b388d8fe831b9a9d75fdae52e5e157 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411210a39bb8a15d54749832801387d5a8599_d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:55,630 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121126d41074a5f463284ddc456b29b7fb4_d7b388d8fe831b9a9d75fdae52e5e157 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121126d41074a5f463284ddc456b29b7fb4_d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:55,631 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212bc92900d35c40e8885c05781789a927_d7b388d8fe831b9a9d75fdae52e5e157 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212bc92900d35c40e8885c05781789a927_d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:55,632 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121451ba96b028442a6b94c1247a0bdd556_d7b388d8fe831b9a9d75fdae52e5e157 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121451ba96b028442a6b94c1247a0bdd556_d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:55,633 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112161b9794f39a84a4b86038fd4251072b0_d7b388d8fe831b9a9d75fdae52e5e157 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112161b9794f39a84a4b86038fd4251072b0_d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:55,634 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216214ba5beaf5447c8d4229fb000f079d_d7b388d8fe831b9a9d75fdae52e5e157 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216214ba5beaf5447c8d4229fb000f079d_d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:55,634 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411217c6b5efbf88b41b897536d37aa602e03_d7b388d8fe831b9a9d75fdae52e5e157 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411217c6b5efbf88b41b897536d37aa602e03_d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:55,635 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112180e74cbe7f2642eeaa1eecd49e2b9ace_d7b388d8fe831b9a9d75fdae52e5e157 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112180e74cbe7f2642eeaa1eecd49e2b9ace_d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:55,636 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112183a5013fdd6c4044819e7518d8b1281d_d7b388d8fe831b9a9d75fdae52e5e157 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112183a5013fdd6c4044819e7518d8b1281d_d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:55,637 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219184a8abe1d3470d8ac68f438d7e9f30_d7b388d8fe831b9a9d75fdae52e5e157 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219184a8abe1d3470d8ac68f438d7e9f30_d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:55,637 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219257cfe14af0414aa47ec3e5a72aac55_d7b388d8fe831b9a9d75fdae52e5e157 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219257cfe14af0414aa47ec3e5a72aac55_d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:55,638 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b5352e45f37e4047977cd21addd9efd0_d7b388d8fe831b9a9d75fdae52e5e157 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b5352e45f37e4047977cd21addd9efd0_d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:55,639 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c1618333cb6f4e6baf57d0e63a16d472_d7b388d8fe831b9a9d75fdae52e5e157 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c1618333cb6f4e6baf57d0e63a16d472_d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:55,640 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d89395f48c5e4604937b38546952af3a_d7b388d8fe831b9a9d75fdae52e5e157 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d89395f48c5e4604937b38546952af3a_d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:55,641 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e704b0bab36547ec8035ad72e5298fbd_d7b388d8fe831b9a9d75fdae52e5e157 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e704b0bab36547ec8035ad72e5298fbd_d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:55,641 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f0abd431549b4799b6db2a4e220936a6_d7b388d8fe831b9a9d75fdae52e5e157 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f0abd431549b4799b6db2a4e220936a6_d7b388d8fe831b9a9d75fdae52e5e157 2024-11-21T11:15:55,642 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T11:15:55,644 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=117, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T11:15:55,646 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T11:15:55,648 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T11:15:55,648 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=117, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T11:15:55,648 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T11:15:55,649 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732187755649"}]},"ts":"9223372036854775807"} 2024-11-21T11:15:55,650 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T11:15:55,650 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d7b388d8fe831b9a9d75fdae52e5e157, NAME => 'TestAcidGuarantees,,1732187722455.d7b388d8fe831b9a9d75fdae52e5e157.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T11:15:55,650 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T11:15:55,651 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732187755650"}]},"ts":"9223372036854775807"} 2024-11-21T11:15:55,652 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T11:15:55,652 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=117, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T11:15:55,653 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 46 msec 2024-11-21T11:15:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-21T11:15:55,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T11:15:55,716 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:15:55,725 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=249 (was 246) - Thread LEAK? -, OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=368 (was 440), ProcessCount=11 (was 11), AvailableMemoryMB=4600 (was 4688) 2024-11-21T11:15:55,734 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=249, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=368, ProcessCount=11, AvailableMemoryMB=4600 2024-11-21T11:15:55,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T11:15:55,735 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T11:15:55,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T11:15:55,737 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T11:15:55,737 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:55,737 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 118 2024-11-21T11:15:55,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-21T11:15:55,738 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T11:15:55,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742280_1456 (size=1061) 2024-11-21T11:15:55,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-21T11:15:56,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-21T11:15:56,145 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322 2024-11-21T11:15:56,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742281_1457 (size=53) 2024-11-21T11:15:56,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-21T11:15:56,550 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:15:56,551 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 20b332191a688c1f969d80830557be9b, disabling compactions & flushes 2024-11-21T11:15:56,551 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:15:56,551 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:15:56,551 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. after waiting 0 ms 2024-11-21T11:15:56,551 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:15:56,551 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:15:56,551 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 20b332191a688c1f969d80830557be9b: Waiting for close lock at 1732187756550Disabling compacts and flushes for region at 1732187756551 (+1 ms)Disabling writes for close at 1732187756551Writing region close event to WAL at 1732187756551Closed at 1732187756551 2024-11-21T11:15:56,552 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T11:15:56,552 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732187756552"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732187756552"}]},"ts":"1732187756552"} 2024-11-21T11:15:56,553 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T11:15:56,554 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T11:15:56,554 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732187756554"}]},"ts":"1732187756554"} 2024-11-21T11:15:56,555 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T11:15:56,555 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=20b332191a688c1f969d80830557be9b, ASSIGN}] 2024-11-21T11:15:56,556 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=20b332191a688c1f969d80830557be9b, ASSIGN 2024-11-21T11:15:56,557 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=20b332191a688c1f969d80830557be9b, ASSIGN; state=OFFLINE, location=7b462513bfc2,33417,1732187618276; forceNewPlan=false, retain=false 2024-11-21T11:15:56,708 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=20b332191a688c1f969d80830557be9b, regionState=OPENING, regionLocation=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:56,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=20b332191a688c1f969d80830557be9b, ASSIGN because future has completed 2024-11-21T11:15:56,709 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure 20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276}] 2024-11-21T11:15:56,864 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:15:56,864 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7752): Opening region: {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} 2024-11-21T11:15:56,864 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 20b332191a688c1f969d80830557be9b 2024-11-21T11:15:56,865 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:15:56,865 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7794): checking encryption for 20b332191a688c1f969d80830557be9b 2024-11-21T11:15:56,865 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7797): checking classloading for 20b332191a688c1f969d80830557be9b 2024-11-21T11:15:56,866 INFO [StoreOpener-20b332191a688c1f969d80830557be9b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 20b332191a688c1f969d80830557be9b 2024-11-21T11:15:56,867 INFO [StoreOpener-20b332191a688c1f969d80830557be9b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T11:15:56,867 INFO [StoreOpener-20b332191a688c1f969d80830557be9b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20b332191a688c1f969d80830557be9b columnFamilyName A 2024-11-21T11:15:56,867 DEBUG [StoreOpener-20b332191a688c1f969d80830557be9b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:56,867 INFO [StoreOpener-20b332191a688c1f969d80830557be9b-1 {}] regionserver.HStore(327): Store=20b332191a688c1f969d80830557be9b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:15:56,867 INFO [StoreOpener-20b332191a688c1f969d80830557be9b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 20b332191a688c1f969d80830557be9b 2024-11-21T11:15:56,868 INFO [StoreOpener-20b332191a688c1f969d80830557be9b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T11:15:56,868 INFO [StoreOpener-20b332191a688c1f969d80830557be9b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20b332191a688c1f969d80830557be9b columnFamilyName B 2024-11-21T11:15:56,868 DEBUG [StoreOpener-20b332191a688c1f969d80830557be9b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:56,869 INFO [StoreOpener-20b332191a688c1f969d80830557be9b-1 {}] regionserver.HStore(327): Store=20b332191a688c1f969d80830557be9b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:15:56,869 INFO [StoreOpener-20b332191a688c1f969d80830557be9b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 20b332191a688c1f969d80830557be9b 2024-11-21T11:15:56,870 INFO [StoreOpener-20b332191a688c1f969d80830557be9b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T11:15:56,870 INFO [StoreOpener-20b332191a688c1f969d80830557be9b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20b332191a688c1f969d80830557be9b columnFamilyName C 2024-11-21T11:15:56,870 DEBUG [StoreOpener-20b332191a688c1f969d80830557be9b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:15:56,870 INFO [StoreOpener-20b332191a688c1f969d80830557be9b-1 {}] regionserver.HStore(327): Store=20b332191a688c1f969d80830557be9b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:15:56,870 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:15:56,870 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1038): replaying wal for 20b332191a688c1f969d80830557be9b 2024-11-21T11:15:56,871 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b 2024-11-21T11:15:56,871 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b 2024-11-21T11:15:56,872 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1048): stopping wal replay for 20b332191a688c1f969d80830557be9b 2024-11-21T11:15:56,872 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1060): Cleaning up temporary data for 20b332191a688c1f969d80830557be9b 2024-11-21T11:15:56,872 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T11:15:56,873 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1093): writing seq id for 20b332191a688c1f969d80830557be9b 2024-11-21T11:15:56,875 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T11:15:56,875 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1114): Opened 20b332191a688c1f969d80830557be9b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70082985, jitterRate=0.044317856431007385}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T11:15:56,875 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 20b332191a688c1f969d80830557be9b 2024-11-21T11:15:56,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-21T11:15:56,876 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1006): Region open journal for 20b332191a688c1f969d80830557be9b: Running coprocessor pre-open hook at 1732187756865Writing region info on filesystem at 1732187756865Initializing all the Stores at 1732187756865Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187756865Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187756865Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187756865Cleaning up temporary data from old regions at 1732187756872 (+7 ms)Running coprocessor post-open hooks at 1732187756875 (+3 ms)Region opened successfully at 1732187756876 (+1 ms) 2024-11-21T11:15:56,876 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., pid=120, masterSystemTime=1732187756861 2024-11-21T11:15:56,878 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:15:56,878 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:15:56,878 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=20b332191a688c1f969d80830557be9b, regionState=OPEN, openSeqNum=2, regionLocation=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:56,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure 20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 because future has completed 2024-11-21T11:15:56,881 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-21T11:15:56,881 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; OpenRegionProcedure 20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 in 171 msec 2024-11-21T11:15:56,882 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-11-21T11:15:56,882 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=20b332191a688c1f969d80830557be9b, ASSIGN in 326 msec 2024-11-21T11:15:56,883 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T11:15:56,883 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732187756883"}]},"ts":"1732187756883"} 2024-11-21T11:15:56,884 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T11:15:56,885 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T11:15:56,886 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1500 sec 2024-11-21T11:15:57,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-21T11:15:57,886 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:15:57,887 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72417d0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:57,887 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:15:57,887 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:15:57,888 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:15:57,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:15:57,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:15:57,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33045730, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:57,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:15:57,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:15:57,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:57,889 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:15:57,890 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76abb731, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:57,890 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:15:57,890 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:15:57,891 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:15:57,891 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:15:57,892 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T11:15:57,892 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 7b462513bfc2,36543,1732187617563 2024-11-21T11:15:57,892 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@8ea1e01 2024-11-21T11:15:57,893 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T11:15:57,893 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T11:15:57,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3692791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:57,894 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:15:57,894 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:15:57,895 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:15:57,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:15:57,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:15:57,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@468ae25c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:57,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:15:57,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:15:57,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:57,896 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:15:57,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a55e52a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:57,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b791feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:57,897 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:15:57,897 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:15:57,897 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:15:57,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:15:57,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:15:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e9d8db1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:15:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:15:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:57,898 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:15:57,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59133630, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:57,899 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da2e0be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:57,899 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:15:57,899 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:15:57,900 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:15:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:15:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:15:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51e4cb36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:15:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:15:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:57,901 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:15:57,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61857e35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:57,902 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc361f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:57,902 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:15:57,902 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:15:57,902 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:15:57,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:15:57,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:15:57,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a7143ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:57,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:15:57,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:15:57,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:57,903 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55976, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:15:57,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@622fa2af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:57,904 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259a56de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:57,904 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:15:57,904 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:15:57,905 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:15:57,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:15:57,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:15:57,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3d2fd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:57,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:15:57,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:15:57,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:57,906 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56000, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:15:57,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6703df19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:57,907 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae789b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:57,907 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:15:57,907 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:15:57,907 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:15:57,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:15:57,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:15:57,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14c72d54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:57,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:15:57,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:15:57,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:57,909 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:15:57,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4657a1bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:57,910 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7b3b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:57,910 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:15:57,910 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:15:57,910 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:15:57,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:15:57,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:15:57,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33b9e47e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:57,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:15:57,911 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:15:57,911 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:57,911 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:15:57,912 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d46a738, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:57,912 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e04f49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:57,912 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:15:57,912 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:15:57,913 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:15:57,913 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:15:57,913 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:15:57,913 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@249b576c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:57,913 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:15:57,913 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:15:57,913 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:57,914 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:15:57,914 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6124b6e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:57,915 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ca199c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:57,915 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:15:57,915 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:15:57,915 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:15:57,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:15:57,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:15:57,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29066bb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:57,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:15:57,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:15:57,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:57,916 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56096, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:15:57,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c8363b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:57,917 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d70caa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:57,917 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:15:57,917 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:15:57,917 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:15:57,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:15:57,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:15:57,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29f56068, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:57,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:15:57,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:15:57,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:15:57,918 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56106, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:15:57,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72f336f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:15:57,919 DEBUG [Thread-2052 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:15:57,919 DEBUG [Thread-2065 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:15:57,920 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:15:57,920 DEBUG [Thread-2060 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:15:57,920 DEBUG [Thread-2069 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:15:57,920 DEBUG [Thread-2056 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:15:57,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:15:57,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:15:57,920 DEBUG [Thread-2058 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:15:57,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:15:57,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:15:57,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:15:57,921 DEBUG [Thread-2067 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:15:57,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:15:57,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:15:57,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:15:57,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-21T11:15:57,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:15:57,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:15:57,922 DEBUG [Thread-2071 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:15:57,922 DEBUG [Thread-2054 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:15:57,922 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:15:57,922 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:15:57,922 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:15:57,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:15:57,923 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:15:57,923 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:15:57,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:15:57,923 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:15:57,923 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:15:57,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:15:57,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:15:57,924 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:15:57,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2] 2024-11-21T11:15:57,925 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51756, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:15:57,925 DEBUG [Thread-2063 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:15:57,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:15:57,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:15:57,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2] 2024-11-21T11:15:57,925 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:15:57,925 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:15:57,926 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:15:57,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-21T11:15:57,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2] 2024-11-21T11:15:57,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2] 2024-11-21T11:15:57,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2] 2024-11-21T11:15:57,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:15:57,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:15:57,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2] 2024-11-21T11:15:57,928 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:15:57,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2] 2024-11-21T11:15:57,929 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:15:57,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:15:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2] 2024-11-21T11:15:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2] 2024-11-21T11:15:57,932 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2] 2024-11-21T11:15:57,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 20b332191a688c1f969d80830557be9b 2024-11-21T11:15:57,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20b332191a688c1f969d80830557be9b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T11:15:57,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=A 2024-11-21T11:15:57,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:57,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=B 2024-11-21T11:15:57,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:57,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=C 2024-11-21T11:15:57,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:15:57,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/fbdff1fd956f44fdac43ab95213b8a8d is 50, key is test_row_0/A:col10/1732187757934/Put/seqid=0 2024-11-21T11:15:57,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742282_1458 (size=14341) 2024-11-21T11:15:57,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1732187757948, max timestamp=1732187757983Num uniques 859; 2024-11-21T11:15:57,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1732187757948, max timestamp=1732187757983Num uniques 859; 2024-11-21T11:15:57,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-21T11:15:57,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-21T11:15:57,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1732187757948, max timestamp=1732187757983Num uniques 859; 2024-11-21T11:15:57,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-21T11:15:58,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:58,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51738 deadline: 1732187818011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:58,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:58,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:58,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187818013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:58,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:58,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187818013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:58,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:58,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:58,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:58,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187818015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:58,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:58,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187818015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:58,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:58,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:58,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-21T11:15:58,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-21T11:15:58,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:15:58,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:15:58,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:15:58,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:58,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:58,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:58,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:58,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51738 deadline: 1732187818125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:58,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:58,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187818125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:58,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:58,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:58,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:58,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187818136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:58,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:58,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187818136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:58,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:58,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:58,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187818136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:58,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:58,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:58,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-21T11:15:58,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:15:58,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:15:58,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:15:58,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:58,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:58,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:58,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-21T11:15:58,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:58,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51738 deadline: 1732187818346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:58,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:58,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:58,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187818346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:58,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:58,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:58,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187818346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:58,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:58,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:58,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187818346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:58,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:58,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:58,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187818346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:58,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:58,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/fbdff1fd956f44fdac43ab95213b8a8d 2024-11-21T11:15:58,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-21T11:15:58,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:15:58,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:15:58,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:15:58,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:58,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:58,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:58,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/0a3d15e49cea4e33b65b279ecf5abc72 is 50, key is test_row_0/B:col10/1732187757934/Put/seqid=0 2024-11-21T11:15:58,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742283_1459 (size=12001) 2024-11-21T11:15:58,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-21T11:15:58,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:15:58,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:15:58,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:15:58,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:58,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:58,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:58,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-21T11:15:58,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:58,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51738 deadline: 1732187818665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:58,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:58,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187818666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:58,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:58,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187818666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:58,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:58,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187818666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:58,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187818666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:58,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-21T11:15:58,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:15:58,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:15:58,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:15:58,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:58,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:58,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:58,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/0a3d15e49cea4e33b65b279ecf5abc72 2024-11-21T11:15:58,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/fb1a187789394b4dbaf5985beda06ddd is 50, key is test_row_0/C:col10/1732187757934/Put/seqid=0 2024-11-21T11:15:58,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742284_1460 (size=12001) 2024-11-21T11:15:58,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-21T11:15:58,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:15:58,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:15:58,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:15:58,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:58,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:58,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:58,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-21T11:15:58,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:15:58,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:15:58,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:15:58,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:58,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:58,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:59,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-21T11:15:59,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-21T11:15:59,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:15:59,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:15:59,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:15:59,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:59,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:59,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:15:59,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:59,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51738 deadline: 1732187819185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:59,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:59,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:59,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187819185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:59,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:59,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:59,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:59,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:59,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187819185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:59,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:59,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:59,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:59,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:59,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187819185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:59,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:59,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:59,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:15:59,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:59,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:59,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187819185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:15:59,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:59,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:59,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:15:59,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:15:59,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/fb1a187789394b4dbaf5985beda06ddd 2024-11-21T11:15:59,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/fbdff1fd956f44fdac43ab95213b8a8d as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/fbdff1fd956f44fdac43ab95213b8a8d 2024-11-21T11:15:59,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/fbdff1fd956f44fdac43ab95213b8a8d, entries=200, sequenceid=12, filesize=14.0 K 2024-11-21T11:15:59,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/0a3d15e49cea4e33b65b279ecf5abc72 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/0a3d15e49cea4e33b65b279ecf5abc72 2024-11-21T11:15:59,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/0a3d15e49cea4e33b65b279ecf5abc72, entries=150, sequenceid=12, filesize=11.7 K 2024-11-21T11:15:59,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/fb1a187789394b4dbaf5985beda06ddd as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/fb1a187789394b4dbaf5985beda06ddd 2024-11-21T11:15:59,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/fb1a187789394b4dbaf5985beda06ddd, entries=150, sequenceid=12, filesize=11.7 K 2024-11-21T11:15:59,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 20b332191a688c1f969d80830557be9b in 1320ms, sequenceid=12, compaction requested=false 2024-11-21T11:15:59,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:15:59,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-21T11:15:59,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:15:59,304 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 20b332191a688c1f969d80830557be9b 3/3 column families, dataSize=134.18 KB heapSize=361.55 KB 2024-11-21T11:15:59,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=A 2024-11-21T11:15:59,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:15:59,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=B 2024-11-21T11:15:59,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:15:59,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=C 2024-11-21T11:15:59,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:15:59,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/3767dba113e34163954c9dbee575a1d5 is 50, key is test_row_0/A:col10/1732187758011/Put/seqid=0 2024-11-21T11:15:59,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742285_1461 (size=12001) 2024-11-21T11:15:59,717 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/3767dba113e34163954c9dbee575a1d5 2024-11-21T11:15:59,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/ebdbee38f0cf48e190c8bf1b9b6c01e3 is 50, key is test_row_0/B:col10/1732187758011/Put/seqid=0 2024-11-21T11:15:59,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742286_1462 (size=12001) 2024-11-21T11:16:00,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-21T11:16:00,130 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/ebdbee38f0cf48e190c8bf1b9b6c01e3 2024-11-21T11:16:00,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/2593bb338e6845b88135fc9adc838f9f is 50, key is test_row_0/C:col10/1732187758011/Put/seqid=0 2024-11-21T11:16:00,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742287_1463 (size=12001) 2024-11-21T11:16:00,146 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/2593bb338e6845b88135fc9adc838f9f 2024-11-21T11:16:00,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/3767dba113e34163954c9dbee575a1d5 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/3767dba113e34163954c9dbee575a1d5 2024-11-21T11:16:00,155 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/3767dba113e34163954c9dbee575a1d5, entries=150, sequenceid=44, filesize=11.7 K 2024-11-21T11:16:00,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/ebdbee38f0cf48e190c8bf1b9b6c01e3 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/ebdbee38f0cf48e190c8bf1b9b6c01e3 2024-11-21T11:16:00,159 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/ebdbee38f0cf48e190c8bf1b9b6c01e3, entries=150, sequenceid=44, filesize=11.7 K 2024-11-21T11:16:00,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/2593bb338e6845b88135fc9adc838f9f as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/2593bb338e6845b88135fc9adc838f9f 2024-11-21T11:16:00,163 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/2593bb338e6845b88135fc9adc838f9f, entries=150, sequenceid=44, filesize=11.7 K 2024-11-21T11:16:00,164 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~361.50 KB/370176, currentSize=0 B/0 for 20b332191a688c1f969d80830557be9b in 860ms, sequenceid=44, compaction requested=false 2024-11-21T11:16:00,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:00,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:00,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-21T11:16:00,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-21T11:16:00,167 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-21T11:16:00,167 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2400 sec 2024-11-21T11:16:00,169 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.2480 sec 2024-11-21T11:16:00,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 20b332191a688c1f969d80830557be9b 2024-11-21T11:16:00,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20b332191a688c1f969d80830557be9b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T11:16:00,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=A 2024-11-21T11:16:00,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:00,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=B 2024-11-21T11:16:00,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:00,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=C 2024-11-21T11:16:00,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:00,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/cc79ab9238de4b9cb5defaa6554fca3d is 50, key is test_row_0/A:col10/1732187760210/Put/seqid=0 2024-11-21T11:16:00,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742288_1464 (size=12001) 2024-11-21T11:16:00,251 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732187760231, max timestamp=1732187760247Num uniques 859; 2024-11-21T11:16:00,251 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-21T11:16:00,251 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732187760231, max timestamp=1732187760247Num uniques 859; 2024-11-21T11:16:00,251 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732187760231, max timestamp=1732187760247Num uniques 859; 2024-11-21T11:16:00,251 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-21T11:16:00,251 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-21T11:16:00,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:00,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51738 deadline: 1732187820278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:00,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:00,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187820280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:00,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187820281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:00,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187820287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:00,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:00,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:00,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187820290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:00,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:00,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:00,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51738 deadline: 1732187820395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:00,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:00,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:00,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187820405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:00,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:00,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187820405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:00,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:00,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:00,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187820405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:00,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:00,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:00,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:00,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187820405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:00,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:00,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:00,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51738 deadline: 1732187820605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:00,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:00,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/cc79ab9238de4b9cb5defaa6554fca3d 2024-11-21T11:16:00,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/6d293e0286934c068393a95ed6c6a867 is 50, key is test_row_0/B:col10/1732187760210/Put/seqid=0 2024-11-21T11:16:00,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742289_1465 (size=12001) 2024-11-21T11:16:00,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:00,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187820625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:00,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:00,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187820625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:00,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:00,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:00,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187820626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:00,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:00,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:00,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187820626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:00,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:00,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:00,824 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T11:16:00,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:00,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51738 deadline: 1732187820925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:00,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:00,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:00,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187820945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:00,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:00,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187820945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:00,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:00,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:00,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:00,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187820946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:00,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:00,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187820946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:00,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:00,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:00,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:01,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/6d293e0286934c068393a95ed6c6a867 2024-11-21T11:16:01,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/ccdfbc30d32f4af6a69655dc80c46c15 is 50, key is test_row_0/C:col10/1732187760210/Put/seqid=0 2024-11-21T11:16:01,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742290_1466 (size=12001) 2024-11-21T11:16:01,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/ccdfbc30d32f4af6a69655dc80c46c15 2024-11-21T11:16:01,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/cc79ab9238de4b9cb5defaa6554fca3d as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/cc79ab9238de4b9cb5defaa6554fca3d 2024-11-21T11:16:01,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:01,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51738 deadline: 1732187821445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:01,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:01,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:01,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:01,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/cc79ab9238de4b9cb5defaa6554fca3d, entries=150, sequenceid=54, filesize=11.7 K 2024-11-21T11:16:01,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/6d293e0286934c068393a95ed6c6a867 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/6d293e0286934c068393a95ed6c6a867 2024-11-21T11:16:01,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/6d293e0286934c068393a95ed6c6a867, entries=150, sequenceid=54, filesize=11.7 K 2024-11-21T11:16:01,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/ccdfbc30d32f4af6a69655dc80c46c15 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/ccdfbc30d32f4af6a69655dc80c46c15 2024-11-21T11:16:01,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/ccdfbc30d32f4af6a69655dc80c46c15, entries=150, sequenceid=54, filesize=11.7 K 2024-11-21T11:16:01,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 20b332191a688c1f969d80830557be9b in 1251ms, sequenceid=54, compaction requested=true 2024-11-21T11:16:01,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:01,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20b332191a688c1f969d80830557be9b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:16:01,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:01,462 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:16:01,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20b332191a688c1f969d80830557be9b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:16:01,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:01,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20b332191a688c1f969d80830557be9b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:16:01,462 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:16:01,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:16:01,463 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:16:01,463 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:16:01,463 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 20b332191a688c1f969d80830557be9b/A is initiating minor compaction (all files) 2024-11-21T11:16:01,463 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 20b332191a688c1f969d80830557be9b/B is initiating minor compaction (all files) 2024-11-21T11:16:01,463 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20b332191a688c1f969d80830557be9b/A in TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:01,463 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20b332191a688c1f969d80830557be9b/B in TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:01,463 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/fbdff1fd956f44fdac43ab95213b8a8d, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/3767dba113e34163954c9dbee575a1d5, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/cc79ab9238de4b9cb5defaa6554fca3d] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp, totalSize=37.4 K 2024-11-21T11:16:01,463 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/0a3d15e49cea4e33b65b279ecf5abc72, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/ebdbee38f0cf48e190c8bf1b9b6c01e3, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/6d293e0286934c068393a95ed6c6a867] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp, totalSize=35.2 K 2024-11-21T11:16:01,463 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbdff1fd956f44fdac43ab95213b8a8d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732187757932 2024-11-21T11:16:01,463 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a3d15e49cea4e33b65b279ecf5abc72, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732187757934 2024-11-21T11:16:01,464 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting ebdbee38f0cf48e190c8bf1b9b6c01e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1732187758011 2024-11-21T11:16:01,464 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3767dba113e34163954c9dbee575a1d5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1732187758011 2024-11-21T11:16:01,464 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc79ab9238de4b9cb5defaa6554fca3d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732187760205 2024-11-21T11:16:01,464 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d293e0286934c068393a95ed6c6a867, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732187760205 2024-11-21T11:16:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 20b332191a688c1f969d80830557be9b 2024-11-21T11:16:01,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20b332191a688c1f969d80830557be9b 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-21T11:16:01,472 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20b332191a688c1f969d80830557be9b#B#compaction#386 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:01,472 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/57cc2fa179c34c17a748ecc2553b06d6 is 50, key is test_row_0/B:col10/1732187760210/Put/seqid=0 2024-11-21T11:16:01,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=A 2024-11-21T11:16:01,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:16:01,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=B 2024-11-21T11:16:01,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:16:01,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=C 2024-11-21T11:16:01,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:16:01,475 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20b332191a688c1f969d80830557be9b#A#compaction#387 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:01,475 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/dd04a7b4837249babe50d448c4f6aede is 50, key is test_row_0/A:col10/1732187760210/Put/seqid=0 2024-11-21T11:16:01,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/5b44e85c8a4d40acbccb62799f6c3def is 50, key is test_row_0/A:col10/1732187760279/Put/seqid=0 2024-11-21T11:16:01,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742291_1467 (size=12104) 2024-11-21T11:16:01,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742292_1468 (size=16681) 2024-11-21T11:16:01,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742293_1469 (size=12104) 2024-11-21T11:16:01,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:01,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:01,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187821493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:01,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187821493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:01,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:01,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187821495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:01,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:01,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:01,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:01,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:01,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:01,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:01,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:01,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:01,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:01,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187821495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:01,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:01,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:01,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:01,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:01,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:01,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187821625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:01,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:01,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:01,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187821625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:01,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:01,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:01,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:01,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:01,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187821625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:01,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:01,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:01,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:01,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:01,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187821626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:01,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:01,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:01,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:01,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:01,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:01,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:01,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187821845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:01,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:01,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187821846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:01,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:01,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187821846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:01,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:01,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187821846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:01,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:01,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:01,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:01,884 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/57cc2fa179c34c17a748ecc2553b06d6 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/57cc2fa179c34c17a748ecc2553b06d6 2024-11-21T11:16:01,888 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20b332191a688c1f969d80830557be9b/B of 20b332191a688c1f969d80830557be9b into 57cc2fa179c34c17a748ecc2553b06d6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:01,888 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:01,888 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., storeName=20b332191a688c1f969d80830557be9b/B, priority=13, startTime=1732187761462; duration=0sec 2024-11-21T11:16:01,888 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:16:01,888 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20b332191a688c1f969d80830557be9b:B 2024-11-21T11:16:01,888 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:16:01,889 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:16:01,889 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 20b332191a688c1f969d80830557be9b/C is initiating minor compaction (all files) 2024-11-21T11:16:01,889 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20b332191a688c1f969d80830557be9b/C in TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:01,889 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/fb1a187789394b4dbaf5985beda06ddd, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/2593bb338e6845b88135fc9adc838f9f, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/ccdfbc30d32f4af6a69655dc80c46c15] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp, totalSize=35.2 K 2024-11-21T11:16:01,889 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting fb1a187789394b4dbaf5985beda06ddd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732187757934 2024-11-21T11:16:01,889 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 2593bb338e6845b88135fc9adc838f9f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1732187758011 2024-11-21T11:16:01,890 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting ccdfbc30d32f4af6a69655dc80c46c15, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732187760205 2024-11-21T11:16:01,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/5b44e85c8a4d40acbccb62799f6c3def 2024-11-21T11:16:01,895 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/dd04a7b4837249babe50d448c4f6aede as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/dd04a7b4837249babe50d448c4f6aede 2024-11-21T11:16:01,900 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20b332191a688c1f969d80830557be9b/A of 20b332191a688c1f969d80830557be9b into dd04a7b4837249babe50d448c4f6aede(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:01,900 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:01,900 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., storeName=20b332191a688c1f969d80830557be9b/A, priority=13, startTime=1732187761462; duration=0sec 2024-11-21T11:16:01,900 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:01,900 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20b332191a688c1f969d80830557be9b:A 2024-11-21T11:16:01,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/00256a54c8af46bbaf0952a74321a657 is 50, key is test_row_0/B:col10/1732187760279/Put/seqid=0 2024-11-21T11:16:01,902 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20b332191a688c1f969d80830557be9b#C#compaction#390 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:01,903 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/bce07a42a8994973bb33d2025761e6af is 50, key is test_row_0/C:col10/1732187760210/Put/seqid=0 2024-11-21T11:16:01,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742294_1470 (size=12001) 2024-11-21T11:16:01,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742295_1471 (size=12104) 2024-11-21T11:16:02,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-21T11:16:02,096 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:16:02,096 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:16:02,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-21T11:16:02,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T11:16:02,098 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:16:02,099 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:16:02,099 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:16:02,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:02,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187822165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:02,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:02,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187822165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:02,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:02,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187822165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:02,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:02,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:02,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187822166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:02,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:02,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:02,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:02,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T11:16:02,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-21T11:16:02,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:02,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:16:02,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:02,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:02,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:02,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:02,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/00256a54c8af46bbaf0952a74321a657 2024-11-21T11:16:02,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/ff9541d8a3814db7ab2808ba00cb5b51 is 50, key is test_row_0/C:col10/1732187760279/Put/seqid=0 2024-11-21T11:16:02,316 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/bce07a42a8994973bb33d2025761e6af as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/bce07a42a8994973bb33d2025761e6af 2024-11-21T11:16:02,320 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20b332191a688c1f969d80830557be9b/C of 20b332191a688c1f969d80830557be9b into bce07a42a8994973bb33d2025761e6af(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:02,321 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:02,321 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., storeName=20b332191a688c1f969d80830557be9b/C, priority=13, startTime=1732187761462; duration=0sec 2024-11-21T11:16:02,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742296_1472 (size=12001) 2024-11-21T11:16:02,321 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:02,321 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20b332191a688c1f969d80830557be9b:C 2024-11-21T11:16:02,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-21T11:16:02,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:02,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:16:02,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:02,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:02,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:02,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:02,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T11:16:02,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:02,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51738 deadline: 1732187822455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:02,456 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_1' in 20b332191a688c1f969d80830557be9b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:16:02,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:02,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:02,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:02,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-21T11:16:02,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:02,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:16:02,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:02,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:02,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:02,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:02,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:02,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187822675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:02,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:02,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:02,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:02,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:02,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187822685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:02,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:02,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:02,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:02,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:02,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187822685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:02,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:02,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:02,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:02,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:02,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187822685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:02,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:02,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:02,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:02,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-21T11:16:02,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:02,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:16:02,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:02,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:02,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:02,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:02,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/ff9541d8a3814db7ab2808ba00cb5b51 2024-11-21T11:16:02,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T11:16:02,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/5b44e85c8a4d40acbccb62799f6c3def as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/5b44e85c8a4d40acbccb62799f6c3def 2024-11-21T11:16:02,729 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/5b44e85c8a4d40acbccb62799f6c3def, entries=250, sequenceid=91, filesize=16.3 K 2024-11-21T11:16:02,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/00256a54c8af46bbaf0952a74321a657 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/00256a54c8af46bbaf0952a74321a657 2024-11-21T11:16:02,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/00256a54c8af46bbaf0952a74321a657, entries=150, sequenceid=91, filesize=11.7 K 2024-11-21T11:16:02,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/ff9541d8a3814db7ab2808ba00cb5b51 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/ff9541d8a3814db7ab2808ba00cb5b51 2024-11-21T11:16:02,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/ff9541d8a3814db7ab2808ba00cb5b51, entries=150, sequenceid=91, filesize=11.7 K 2024-11-21T11:16:02,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~430.64 KB/440976, currentSize=40.25 KB/41220 for 20b332191a688c1f969d80830557be9b in 1266ms, sequenceid=91, compaction requested=false 2024-11-21T11:16:02,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:02,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-21T11:16:02,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:02,863 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 20b332191a688c1f969d80830557be9b 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-21T11:16:02,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=A 2024-11-21T11:16:02,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:02,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=B 2024-11-21T11:16:02,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:02,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=C 2024-11-21T11:16:02,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:02,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/a3d5a463df1943cdb8f307706bffe136 is 50, key is test_row_0/A:col10/1732187761493/Put/seqid=0 2024-11-21T11:16:02,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742297_1473 (size=12001) 2024-11-21T11:16:02,872 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/a3d5a463df1943cdb8f307706bffe136 2024-11-21T11:16:02,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/f062b96f97194c50a2b5fd2642e4e1c5 is 50, key is test_row_0/B:col10/1732187761493/Put/seqid=0 2024-11-21T11:16:02,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742298_1474 (size=12001) 2024-11-21T11:16:03,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T11:16:03,284 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/f062b96f97194c50a2b5fd2642e4e1c5 2024-11-21T11:16:03,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/40948d7709a1423abada52037752ddb3 is 50, key is test_row_0/C:col10/1732187761493/Put/seqid=0 2024-11-21T11:16:03,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742299_1475 (size=12001) 2024-11-21T11:16:03,697 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/40948d7709a1423abada52037752ddb3 2024-11-21T11:16:03,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 20b332191a688c1f969d80830557be9b 2024-11-21T11:16:03,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:16:03,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/a3d5a463df1943cdb8f307706bffe136 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/a3d5a463df1943cdb8f307706bffe136 2024-11-21T11:16:03,720 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/a3d5a463df1943cdb8f307706bffe136, entries=150, sequenceid=103, filesize=11.7 K 2024-11-21T11:16:03,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/f062b96f97194c50a2b5fd2642e4e1c5 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/f062b96f97194c50a2b5fd2642e4e1c5 2024-11-21T11:16:03,724 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/f062b96f97194c50a2b5fd2642e4e1c5, entries=150, sequenceid=103, filesize=11.7 K 2024-11-21T11:16:03,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/40948d7709a1423abada52037752ddb3 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/40948d7709a1423abada52037752ddb3 2024-11-21T11:16:03,729 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/40948d7709a1423abada52037752ddb3, entries=150, sequenceid=103, filesize=11.7 K 2024-11-21T11:16:03,730 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=80.51 KB/82440 for 20b332191a688c1f969d80830557be9b in 867ms, sequenceid=103, compaction requested=true 2024-11-21T11:16:03,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:03,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:03,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-21T11:16:03,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 20b332191a688c1f969d80830557be9b 2024-11-21T11:16:03,731 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20b332191a688c1f969d80830557be9b 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-21T11:16:03,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-21T11:16:03,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=A 2024-11-21T11:16:03,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:03,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=B 2024-11-21T11:16:03,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:03,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=C 2024-11-21T11:16:03,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:03,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-21T11:16:03,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6330 sec 2024-11-21T11:16:03,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/e5b0746a695044b99babda4fea57b588 is 50, key is test_row_0/A:col10/1732187763729/Put/seqid=0 2024-11-21T11:16:03,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.6390 sec 2024-11-21T11:16:03,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742300_1476 (size=16681) 2024-11-21T11:16:03,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:03,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187823759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:03,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:03,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:03,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:03,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:03,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187823760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:03,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:03,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:03,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:03,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:03,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187823765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:03,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:03,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:03,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:03,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187823766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:03,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:03,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:03,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:03,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:03,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:03,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187823875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:03,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:03,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:03,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:03,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:03,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187823885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:03,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:03,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:03,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187823885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:03,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:03,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:03,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:03,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:03,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187823885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:03,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:03,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:03,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:03,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:03,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:04,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:04,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187824095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:04,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:04,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187824096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:04,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:04,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:04,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187824096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:04,101 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:04,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:04,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:04,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187824096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:04,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:04,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:04,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:04,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:04,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:04,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:04,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:04,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:04,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:04,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/e5b0746a695044b99babda4fea57b588 2024-11-21T11:16:04,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/066c544e311147e6939659de48f90b83 is 50, key is test_row_0/B:col10/1732187763729/Put/seqid=0 2024-11-21T11:16:04,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742301_1477 (size=12001) 2024-11-21T11:16:04,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/066c544e311147e6939659de48f90b83 2024-11-21T11:16:04,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/b697fc067ef440588804e4017605727f is 50, key is test_row_0/C:col10/1732187763729/Put/seqid=0 2024-11-21T11:16:04,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742302_1478 (size=12001) 2024-11-21T11:16:04,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T11:16:04,246 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:16:04,247 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:16:04,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-21T11:16:04,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T11:16:04,248 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:16:04,249 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:16:04,249 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:16:04,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T11:16:04,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-21T11:16:04,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:04,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:16:04,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:04,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:04,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:04,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:04,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:04,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187824405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:04,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:04,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:04,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:04,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187824405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:04,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:04,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:04,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:04,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187824405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:04,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:04,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:04,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:04,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:04,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:04,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:04,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187824406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:04,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:04,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:04,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:04,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:04,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51738 deadline: 1732187824465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:04,474 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_1' in 20b332191a688c1f969d80830557be9b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:16:04,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:04,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:04,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:04,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-21T11:16:04,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:04,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:16:04,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:04,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:04,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:04,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:04,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T11:16:04,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/b697fc067ef440588804e4017605727f 2024-11-21T11:16:04,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/e5b0746a695044b99babda4fea57b588 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/e5b0746a695044b99babda4fea57b588 2024-11-21T11:16:04,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/e5b0746a695044b99babda4fea57b588, entries=250, sequenceid=119, filesize=16.3 K 2024-11-21T11:16:04,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/066c544e311147e6939659de48f90b83 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/066c544e311147e6939659de48f90b83 2024-11-21T11:16:04,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/066c544e311147e6939659de48f90b83, entries=150, sequenceid=119, filesize=11.7 K 2024-11-21T11:16:04,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/b697fc067ef440588804e4017605727f as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/b697fc067ef440588804e4017605727f 2024-11-21T11:16:04,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/b697fc067ef440588804e4017605727f, entries=150, sequenceid=119, filesize=11.7 K 2024-11-21T11:16:04,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 20b332191a688c1f969d80830557be9b in 853ms, sequenceid=119, compaction requested=true 2024-11-21T11:16:04,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:04,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20b332191a688c1f969d80830557be9b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:16:04,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:04,584 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T11:16:04,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20b332191a688c1f969d80830557be9b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:16:04,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:04,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20b332191a688c1f969d80830557be9b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:16:04,584 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T11:16:04,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:16:04,585 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57467 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T11:16:04,585 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 20b332191a688c1f969d80830557be9b/A is initiating minor compaction (all files) 2024-11-21T11:16:04,585 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20b332191a688c1f969d80830557be9b/A in TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:04,585 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/dd04a7b4837249babe50d448c4f6aede, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/5b44e85c8a4d40acbccb62799f6c3def, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/a3d5a463df1943cdb8f307706bffe136, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/e5b0746a695044b99babda4fea57b588] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp, totalSize=56.1 K 2024-11-21T11:16:04,586 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T11:16:04,586 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 20b332191a688c1f969d80830557be9b/B is initiating minor compaction (all files) 2024-11-21T11:16:04,586 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20b332191a688c1f969d80830557be9b/B in TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:04,586 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/57cc2fa179c34c17a748ecc2553b06d6, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/00256a54c8af46bbaf0952a74321a657, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/f062b96f97194c50a2b5fd2642e4e1c5, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/066c544e311147e6939659de48f90b83] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp, totalSize=47.0 K 2024-11-21T11:16:04,586 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd04a7b4837249babe50d448c4f6aede, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732187760205 2024-11-21T11:16:04,586 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 57cc2fa179c34c17a748ecc2553b06d6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732187760205 2024-11-21T11:16:04,586 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b44e85c8a4d40acbccb62799f6c3def, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732187760279 2024-11-21T11:16:04,586 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 00256a54c8af46bbaf0952a74321a657, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732187760279 2024-11-21T11:16:04,587 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting f062b96f97194c50a2b5fd2642e4e1c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732187761472 2024-11-21T11:16:04,587 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3d5a463df1943cdb8f307706bffe136, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732187761472 2024-11-21T11:16:04,587 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 066c544e311147e6939659de48f90b83, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732187763726 2024-11-21T11:16:04,587 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5b0746a695044b99babda4fea57b588, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732187763710 2024-11-21T11:16:04,596 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20b332191a688c1f969d80830557be9b#B#compaction#398 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:04,596 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20b332191a688c1f969d80830557be9b#A#compaction#399 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:04,597 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/b76c10047a8142678e639a92143cf523 is 50, key is test_row_0/B:col10/1732187763729/Put/seqid=0 2024-11-21T11:16:04,597 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/830e4a5db83f43d8b4c96f64e04e83cd is 50, key is test_row_0/A:col10/1732187763729/Put/seqid=0 2024-11-21T11:16:04,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742304_1480 (size=12241) 2024-11-21T11:16:04,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742303_1479 (size=12241) 2024-11-21T11:16:04,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-21T11:16:04,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:04,708 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 20b332191a688c1f969d80830557be9b 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-21T11:16:04,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=A 2024-11-21T11:16:04,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:04,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=B 2024-11-21T11:16:04,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:04,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=C 2024-11-21T11:16:04,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:04,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/5b778ca750704fdea3f04969692592c1 is 50, key is test_row_0/A:col10/1732187763759/Put/seqid=0 2024-11-21T11:16:04,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742305_1481 (size=12151) 2024-11-21T11:16:04,719 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/5b778ca750704fdea3f04969692592c1 2024-11-21T11:16:04,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/e316d953f0974427b509dd2821cdee6e is 50, key is test_row_0/B:col10/1732187763759/Put/seqid=0 2024-11-21T11:16:04,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742306_1482 (size=12151) 2024-11-21T11:16:04,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T11:16:04,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 20b332191a688c1f969d80830557be9b 2024-11-21T11:16:04,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:16:04,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:04,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187824955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:04,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:04,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:04,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:04,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187824966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:04,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:04,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:04,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:04,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:04,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:04,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187824967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:04,979 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:04,979 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:04,979 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:04,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:04,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187824968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:04,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:04,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:04,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:05,007 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/830e4a5db83f43d8b4c96f64e04e83cd as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/830e4a5db83f43d8b4c96f64e04e83cd 2024-11-21T11:16:05,010 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/b76c10047a8142678e639a92143cf523 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/b76c10047a8142678e639a92143cf523 2024-11-21T11:16:05,012 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 20b332191a688c1f969d80830557be9b/A of 20b332191a688c1f969d80830557be9b into 830e4a5db83f43d8b4c96f64e04e83cd(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:05,012 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:05,012 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., storeName=20b332191a688c1f969d80830557be9b/A, priority=12, startTime=1732187764584; duration=0sec 2024-11-21T11:16:05,013 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:16:05,013 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20b332191a688c1f969d80830557be9b:A 2024-11-21T11:16:05,013 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T11:16:05,014 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T11:16:05,014 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 20b332191a688c1f969d80830557be9b/C is initiating minor compaction (all files) 2024-11-21T11:16:05,014 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20b332191a688c1f969d80830557be9b/C in TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:05,014 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/bce07a42a8994973bb33d2025761e6af, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/ff9541d8a3814db7ab2808ba00cb5b51, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/40948d7709a1423abada52037752ddb3, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/b697fc067ef440588804e4017605727f] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp, totalSize=47.0 K 2024-11-21T11:16:05,015 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting bce07a42a8994973bb33d2025761e6af, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732187760205 2024-11-21T11:16:05,015 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff9541d8a3814db7ab2808ba00cb5b51, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732187760279 2024-11-21T11:16:05,015 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40948d7709a1423abada52037752ddb3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732187761472 2024-11-21T11:16:05,015 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting b697fc067ef440588804e4017605727f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732187763726 2024-11-21T11:16:05,022 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 20b332191a688c1f969d80830557be9b/B of 20b332191a688c1f969d80830557be9b into b76c10047a8142678e639a92143cf523(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:05,022 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:05,022 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., storeName=20b332191a688c1f969d80830557be9b/B, priority=12, startTime=1732187764584; duration=0sec 2024-11-21T11:16:05,022 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:05,022 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20b332191a688c1f969d80830557be9b:B 2024-11-21T11:16:05,023 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20b332191a688c1f969d80830557be9b#C#compaction#402 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:05,023 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/23fe4ed45fc0465b8a4ebef2f3e2d6c8 is 50, key is test_row_0/C:col10/1732187763729/Put/seqid=0 2024-11-21T11:16:05,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742307_1483 (size=12241) 2024-11-21T11:16:05,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:05,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187825075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:05,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:05,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187825075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:05,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:05,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:05,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:05,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187825086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:05,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:05,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:05,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187825095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:05,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:05,131 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/e316d953f0974427b509dd2821cdee6e 2024-11-21T11:16:05,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/85a389aaa2e7479fb8520a0a4c6bd675 is 50, key is test_row_0/C:col10/1732187763759/Put/seqid=0 2024-11-21T11:16:05,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742308_1484 (size=12151) 2024-11-21T11:16:05,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:05,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187825285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:05,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:05,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187825285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:05,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:05,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:05,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:05,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187825306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:05,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:05,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187825315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:05,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:05,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T11:16:05,438 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/23fe4ed45fc0465b8a4ebef2f3e2d6c8 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/23fe4ed45fc0465b8a4ebef2f3e2d6c8 2024-11-21T11:16:05,442 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 20b332191a688c1f969d80830557be9b/C of 20b332191a688c1f969d80830557be9b into 23fe4ed45fc0465b8a4ebef2f3e2d6c8(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:05,442 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:05,442 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., storeName=20b332191a688c1f969d80830557be9b/C, priority=12, startTime=1732187764584; duration=0sec 2024-11-21T11:16:05,442 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:05,443 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20b332191a688c1f969d80830557be9b:C 2024-11-21T11:16:05,567 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/85a389aaa2e7479fb8520a0a4c6bd675 2024-11-21T11:16:05,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/5b778ca750704fdea3f04969692592c1 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/5b778ca750704fdea3f04969692592c1 2024-11-21T11:16:05,574 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/5b778ca750704fdea3f04969692592c1, entries=150, sequenceid=135, filesize=11.9 K 2024-11-21T11:16:05,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/e316d953f0974427b509dd2821cdee6e as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/e316d953f0974427b509dd2821cdee6e 2024-11-21T11:16:05,578 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/e316d953f0974427b509dd2821cdee6e, entries=150, sequenceid=135, filesize=11.9 K 2024-11-21T11:16:05,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/85a389aaa2e7479fb8520a0a4c6bd675 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/85a389aaa2e7479fb8520a0a4c6bd675 2024-11-21T11:16:05,582 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/85a389aaa2e7479fb8520a0a4c6bd675, entries=150, sequenceid=135, filesize=11.9 K 2024-11-21T11:16:05,583 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 20b332191a688c1f969d80830557be9b in 875ms, sequenceid=135, compaction requested=false 2024-11-21T11:16:05,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:05,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:05,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-21T11:16:05,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-21T11:16:05,585 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-21T11:16:05,585 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3350 sec 2024-11-21T11:16:05,587 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.3390 sec 2024-11-21T11:16:05,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 20b332191a688c1f969d80830557be9b 2024-11-21T11:16:05,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20b332191a688c1f969d80830557be9b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-21T11:16:05,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=A 2024-11-21T11:16:05,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:05,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=B 2024-11-21T11:16:05,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:05,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=C 2024-11-21T11:16:05,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:05,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/578a195ab9614f50b44ca1150f658f60 is 50, key is test_row_0/A:col10/1732187764965/Put/seqid=0 2024-11-21T11:16:05,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742309_1485 (size=14541) 2024-11-21T11:16:05,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:05,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187825638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:05,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:05,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187825647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:05,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:05,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:05,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187825648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:05,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:05,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187825649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:05,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:05,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:05,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187825755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:05,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187825755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:05,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:05,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187825765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:05,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:05,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187825765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:05,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:05,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:05,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187825975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:05,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:05,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187825975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:05,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:05,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:05,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187825975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:05,984 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,984 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,984 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:05,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187825985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:05,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:05,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:06,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/578a195ab9614f50b44ca1150f658f60 2024-11-21T11:16:06,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/6282209cf24b4e9c914b592093841257 is 50, key is test_row_0/B:col10/1732187764965/Put/seqid=0 2024-11-21T11:16:06,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742310_1486 (size=12151) 2024-11-21T11:16:06,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187826296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:06,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:06,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:06,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187826296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:06,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:06,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:06,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:06,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187826296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:06,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:06,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:06,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:06,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:06,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187826305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:06,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:06,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:06,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T11:16:06,395 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:16:06,396 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:16:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-21T11:16:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T11:16:06,397 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:16:06,398 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:16:06,398 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:16:06,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/6282209cf24b4e9c914b592093841257 2024-11-21T11:16:06,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/ceef9f7cc69c4653a62fe1a6a9e3d97d is 50, key is test_row_0/C:col10/1732187764965/Put/seqid=0 2024-11-21T11:16:06,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742311_1487 (size=12151) 2024-11-21T11:16:06,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T11:16:06,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-21T11:16:06,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:06,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:16:06,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:06,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:06,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:06,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:06,656 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T11:16:06,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-21T11:16:06,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:06,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:16:06,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:06,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:06,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:06,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:06,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T11:16:06,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:06,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187826815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:06,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:06,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:06,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:06,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187826815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:06,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:06,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:06,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187826816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:06,820 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:06,820 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:06,820 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:06,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:06,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187826816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:06,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:06,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:06,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:06,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:06,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:06,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:06,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/ceef9f7cc69c4653a62fe1a6a9e3d97d 2024-11-21T11:16:06,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/578a195ab9614f50b44ca1150f658f60 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/578a195ab9614f50b44ca1150f658f60 2024-11-21T11:16:06,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/578a195ab9614f50b44ca1150f658f60, entries=200, sequenceid=155, filesize=14.2 K 2024-11-21T11:16:06,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/6282209cf24b4e9c914b592093841257 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/6282209cf24b4e9c914b592093841257 2024-11-21T11:16:06,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/6282209cf24b4e9c914b592093841257, entries=150, sequenceid=155, filesize=11.9 K 2024-11-21T11:16:06,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/ceef9f7cc69c4653a62fe1a6a9e3d97d as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/ceef9f7cc69c4653a62fe1a6a9e3d97d 2024-11-21T11:16:06,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/ceef9f7cc69c4653a62fe1a6a9e3d97d, entries=150, sequenceid=155, filesize=11.9 K 2024-11-21T11:16:06,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-21T11:16:06,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:06,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:16:06,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:06,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:06,857 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 20b332191a688c1f969d80830557be9b in 1247ms, sequenceid=155, compaction requested=true 2024-11-21T11:16:06,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:06,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:06,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20b332191a688c1f969d80830557be9b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:16:06,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:06,858 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:16:06,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20b332191a688c1f969d80830557be9b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:16:06,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:06,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20b332191a688c1f969d80830557be9b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:16:06,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:16:06,858 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:16:06,858 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38933 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:16:06,859 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 20b332191a688c1f969d80830557be9b/A is initiating minor compaction (all files) 2024-11-21T11:16:06,859 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20b332191a688c1f969d80830557be9b/A in TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:06,859 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/830e4a5db83f43d8b4c96f64e04e83cd, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/5b778ca750704fdea3f04969692592c1, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/578a195ab9614f50b44ca1150f658f60] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp, totalSize=38.0 K 2024-11-21T11:16:06,859 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:16:06,859 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 20b332191a688c1f969d80830557be9b/B is initiating minor compaction (all files) 2024-11-21T11:16:06,859 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20b332191a688c1f969d80830557be9b/B in TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:06,859 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/b76c10047a8142678e639a92143cf523, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/e316d953f0974427b509dd2821cdee6e, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/6282209cf24b4e9c914b592093841257] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp, totalSize=35.7 K 2024-11-21T11:16:06,859 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 830e4a5db83f43d8b4c96f64e04e83cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732187763726 2024-11-21T11:16:06,860 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting b76c10047a8142678e639a92143cf523, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732187763726 2024-11-21T11:16:06,860 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b778ca750704fdea3f04969692592c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732187763759 2024-11-21T11:16:06,860 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting e316d953f0974427b509dd2821cdee6e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732187763759 2024-11-21T11:16:06,860 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 6282209cf24b4e9c914b592093841257, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732187764965 2024-11-21T11:16:06,860 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 578a195ab9614f50b44ca1150f658f60, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732187764949 2024-11-21T11:16:06,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:06,868 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20b332191a688c1f969d80830557be9b#B#compaction#407 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:06,868 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/fdd5e8bdafeb43e4a641d34629fc0f17 is 50, key is test_row_0/B:col10/1732187764965/Put/seqid=0 2024-11-21T11:16:06,869 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20b332191a688c1f969d80830557be9b#A#compaction#408 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:06,869 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/3fe50ccdda6b4dbeb4c13b1883dea816 is 50, key is test_row_0/A:col10/1732187764965/Put/seqid=0 2024-11-21T11:16:06,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742312_1488 (size=12493) 2024-11-21T11:16:06,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742313_1489 (size=12493) 2024-11-21T11:16:06,888 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/3fe50ccdda6b4dbeb4c13b1883dea816 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/3fe50ccdda6b4dbeb4c13b1883dea816 2024-11-21T11:16:06,892 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20b332191a688c1f969d80830557be9b/A of 20b332191a688c1f969d80830557be9b into 3fe50ccdda6b4dbeb4c13b1883dea816(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:06,892 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:06,892 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., storeName=20b332191a688c1f969d80830557be9b/A, priority=13, startTime=1732187766858; duration=0sec 2024-11-21T11:16:06,892 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:16:06,892 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20b332191a688c1f969d80830557be9b:A 2024-11-21T11:16:06,892 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:16:06,893 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:16:06,894 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 20b332191a688c1f969d80830557be9b/C is initiating minor compaction (all files) 2024-11-21T11:16:06,894 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20b332191a688c1f969d80830557be9b/C in TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:06,894 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/23fe4ed45fc0465b8a4ebef2f3e2d6c8, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/85a389aaa2e7479fb8520a0a4c6bd675, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/ceef9f7cc69c4653a62fe1a6a9e3d97d] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp, totalSize=35.7 K 2024-11-21T11:16:06,894 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23fe4ed45fc0465b8a4ebef2f3e2d6c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732187763726 2024-11-21T11:16:06,894 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85a389aaa2e7479fb8520a0a4c6bd675, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732187763759 2024-11-21T11:16:06,894 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting ceef9f7cc69c4653a62fe1a6a9e3d97d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732187764965 2024-11-21T11:16:06,901 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20b332191a688c1f969d80830557be9b#C#compaction#409 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:06,901 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/536829c7a7f546ada70b42e40274726c is 50, key is test_row_0/C:col10/1732187764965/Put/seqid=0 2024-11-21T11:16:06,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742314_1490 (size=12493) 2024-11-21T11:16:07,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-21T11:16:07,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:07,015 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 20b332191a688c1f969d80830557be9b 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-21T11:16:07,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=A 2024-11-21T11:16:07,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:07,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=B 2024-11-21T11:16:07,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:07,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=C 2024-11-21T11:16:07,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:07,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/9c9e8258c950494b916f0789c4e1355a is 50, key is test_row_0/A:col10/1732187765647/Put/seqid=0 2024-11-21T11:16:07,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742315_1491 (size=12151) 2024-11-21T11:16:07,024 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/9c9e8258c950494b916f0789c4e1355a 2024-11-21T11:16:07,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T11:16:07,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/55d6d212d46b489d909013db980dbcd2 is 50, key is test_row_0/B:col10/1732187765647/Put/seqid=0 2024-11-21T11:16:07,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742316_1492 (size=12151) 2024-11-21T11:16:07,287 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/fdd5e8bdafeb43e4a641d34629fc0f17 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/fdd5e8bdafeb43e4a641d34629fc0f17 2024-11-21T11:16:07,296 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20b332191a688c1f969d80830557be9b/B of 20b332191a688c1f969d80830557be9b into fdd5e8bdafeb43e4a641d34629fc0f17(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:07,296 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:07,296 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., storeName=20b332191a688c1f969d80830557be9b/B, priority=13, startTime=1732187766858; duration=0sec 2024-11-21T11:16:07,296 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:07,296 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20b332191a688c1f969d80830557be9b:B 2024-11-21T11:16:07,309 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/536829c7a7f546ada70b42e40274726c as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/536829c7a7f546ada70b42e40274726c 2024-11-21T11:16:07,313 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20b332191a688c1f969d80830557be9b/C of 20b332191a688c1f969d80830557be9b into 536829c7a7f546ada70b42e40274726c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:07,313 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:07,313 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., storeName=20b332191a688c1f969d80830557be9b/C, priority=13, startTime=1732187766858; duration=0sec 2024-11-21T11:16:07,313 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:07,313 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20b332191a688c1f969d80830557be9b:C 2024-11-21T11:16:07,440 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/55d6d212d46b489d909013db980dbcd2 2024-11-21T11:16:07,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/801589f6a695477ab917884677363dce is 50, key is test_row_0/C:col10/1732187765647/Put/seqid=0 2024-11-21T11:16:07,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742317_1493 (size=12151) 2024-11-21T11:16:07,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T11:16:07,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 20b332191a688c1f969d80830557be9b 2024-11-21T11:16:07,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:16:07,852 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/801589f6a695477ab917884677363dce 2024-11-21T11:16:07,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/9c9e8258c950494b916f0789c4e1355a as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/9c9e8258c950494b916f0789c4e1355a 2024-11-21T11:16:07,859 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/9c9e8258c950494b916f0789c4e1355a, entries=150, sequenceid=171, filesize=11.9 K 2024-11-21T11:16:07,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/55d6d212d46b489d909013db980dbcd2 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/55d6d212d46b489d909013db980dbcd2 2024-11-21T11:16:07,862 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/55d6d212d46b489d909013db980dbcd2, entries=150, sequenceid=171, filesize=11.9 K 2024-11-21T11:16:07,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/801589f6a695477ab917884677363dce as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/801589f6a695477ab917884677363dce 2024-11-21T11:16:07,866 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/801589f6a695477ab917884677363dce, entries=150, sequenceid=171, filesize=11.9 K 2024-11-21T11:16:07,868 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=40.25 KB/41220 for 20b332191a688c1f969d80830557be9b in 852ms, sequenceid=171, compaction requested=false 2024-11-21T11:16:07,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:07,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:07,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-21T11:16:07,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 20b332191a688c1f969d80830557be9b 2024-11-21T11:16:07,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-21T11:16:07,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20b332191a688c1f969d80830557be9b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-21T11:16:07,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=A 2024-11-21T11:16:07,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:07,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=B 2024-11-21T11:16:07,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:07,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=C 2024-11-21T11:16:07,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:07,871 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-21T11:16:07,872 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4720 sec 2024-11-21T11:16:07,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/cd3bab54c63445dfa8be79c7578c26aa is 50, key is test_row_0/A:col10/1732187767852/Put/seqid=0 2024-11-21T11:16:07,873 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.4760 sec 2024-11-21T11:16:07,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742318_1494 (size=14541) 2024-11-21T11:16:07,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:07,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187827912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:07,919 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:07,919 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:07,919 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:07,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:07,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187827913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:07,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:07,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:07,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:07,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:07,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187827917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:07,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:07,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187827918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:08,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:08,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187828025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:08,033 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:08,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:08,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:08,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:08,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187828035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:08,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:08,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:08,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:08,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:08,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187828046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:08,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:08,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:08,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:08,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:08,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187828046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:08,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:08,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:08,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:08,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:08,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187828245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:08,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:08,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:08,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:08,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:08,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187828256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:08,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:08,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:08,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:08,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:08,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187828265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:08,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:08,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187828265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:08,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:08,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:08,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:08,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:08,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:08,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:08,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/cd3bab54c63445dfa8be79c7578c26aa 2024-11-21T11:16:08,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/8772c60a91024409a9ece10ad6471d31 is 50, key is test_row_0/B:col10/1732187767852/Put/seqid=0 2024-11-21T11:16:08,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742319_1495 (size=12151) 2024-11-21T11:16:08,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/8772c60a91024409a9ece10ad6471d31 2024-11-21T11:16:08,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/5d61fdcd11064d02a858a79487d5bf41 is 50, key is test_row_0/C:col10/1732187767852/Put/seqid=0 2024-11-21T11:16:08,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742320_1496 (size=12151) 2024-11-21T11:16:08,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:08,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51738 deadline: 1732187828496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:08,498 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_1' in 20b332191a688c1f969d80830557be9b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:16:08,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:08,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:08,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:08,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T11:16:08,545 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:16:08,546 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:16:08,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-21T11:16:08,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T11:16:08,548 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:16:08,548 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:16:08,549 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:16:08,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:08,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187828555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:08,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:08,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:08,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:08,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:08,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187828576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:08,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:08,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:08,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187828576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:08,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:08,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:08,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:08,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:08,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:08,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187828576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:08,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:08,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:08,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:08,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:08,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T11:16:08,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T11:16:08,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:08,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:16:08,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:08,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:08,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:08,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:08,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/5d61fdcd11064d02a858a79487d5bf41 2024-11-21T11:16:08,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/cd3bab54c63445dfa8be79c7578c26aa as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/cd3bab54c63445dfa8be79c7578c26aa 2024-11-21T11:16:08,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/cd3bab54c63445dfa8be79c7578c26aa, entries=200, sequenceid=185, filesize=14.2 K 2024-11-21T11:16:08,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/8772c60a91024409a9ece10ad6471d31 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/8772c60a91024409a9ece10ad6471d31 2024-11-21T11:16:08,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/8772c60a91024409a9ece10ad6471d31, entries=150, sequenceid=185, filesize=11.9 K 2024-11-21T11:16:08,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/5d61fdcd11064d02a858a79487d5bf41 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/5d61fdcd11064d02a858a79487d5bf41 2024-11-21T11:16:08,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/5d61fdcd11064d02a858a79487d5bf41, entries=150, sequenceid=185, filesize=11.9 K 2024-11-21T11:16:08,735 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 20b332191a688c1f969d80830557be9b in 867ms, sequenceid=185, compaction requested=true 2024-11-21T11:16:08,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:08,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20b332191a688c1f969d80830557be9b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:16:08,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:08,735 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:16:08,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20b332191a688c1f969d80830557be9b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:16:08,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:08,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20b332191a688c1f969d80830557be9b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:16:08,735 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:16:08,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:16:08,736 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:16:08,736 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:16:08,736 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 20b332191a688c1f969d80830557be9b/A is initiating minor compaction (all files) 2024-11-21T11:16:08,736 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 20b332191a688c1f969d80830557be9b/B is initiating minor compaction (all files) 2024-11-21T11:16:08,736 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20b332191a688c1f969d80830557be9b/B in TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:08,736 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20b332191a688c1f969d80830557be9b/A in TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:08,736 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/fdd5e8bdafeb43e4a641d34629fc0f17, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/55d6d212d46b489d909013db980dbcd2, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/8772c60a91024409a9ece10ad6471d31] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp, totalSize=35.9 K 2024-11-21T11:16:08,736 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/3fe50ccdda6b4dbeb4c13b1883dea816, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/9c9e8258c950494b916f0789c4e1355a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/cd3bab54c63445dfa8be79c7578c26aa] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp, totalSize=38.3 K 2024-11-21T11:16:08,737 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting fdd5e8bdafeb43e4a641d34629fc0f17, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732187764965 2024-11-21T11:16:08,737 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fe50ccdda6b4dbeb4c13b1883dea816, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732187764965 2024-11-21T11:16:08,739 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c9e8258c950494b916f0789c4e1355a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732187765636 2024-11-21T11:16:08,739 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 55d6d212d46b489d909013db980dbcd2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732187765636 2024-11-21T11:16:08,739 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd3bab54c63445dfa8be79c7578c26aa, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732187767840 2024-11-21T11:16:08,739 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 8772c60a91024409a9ece10ad6471d31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732187767840 2024-11-21T11:16:08,746 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20b332191a688c1f969d80830557be9b#B#compaction#416 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:08,746 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/5b4bdca400a84c8aaece569d4622bd8f is 50, key is test_row_0/B:col10/1732187767852/Put/seqid=0 2024-11-21T11:16:08,750 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20b332191a688c1f969d80830557be9b#A#compaction#417 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:08,750 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/b1c642677b17427f96a946fba4791682 is 50, key is test_row_0/A:col10/1732187767852/Put/seqid=0 2024-11-21T11:16:08,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742321_1497 (size=12595) 2024-11-21T11:16:08,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742322_1498 (size=12595) 2024-11-21T11:16:08,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T11:16:08,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:08,854 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 20b332191a688c1f969d80830557be9b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-21T11:16:08,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=A 2024-11-21T11:16:08,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:08,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=B 2024-11-21T11:16:08,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:08,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=C 2024-11-21T11:16:08,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:08,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/777d4b4e6cb04cf9b0e38518bac0c39e is 50, key is test_row_0/A:col10/1732187767892/Put/seqid=0 2024-11-21T11:16:08,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742323_1499 (size=12151) 2024-11-21T11:16:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T11:16:09,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 20b332191a688c1f969d80830557be9b 2024-11-21T11:16:09,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:16:09,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:09,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187829095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:09,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:09,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:09,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187829104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:09,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:09,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:09,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187829105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:09,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:09,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:09,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187829106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:09,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:09,157 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/5b4bdca400a84c8aaece569d4622bd8f as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/5b4bdca400a84c8aaece569d4622bd8f 2024-11-21T11:16:09,160 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/b1c642677b17427f96a946fba4791682 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/b1c642677b17427f96a946fba4791682 2024-11-21T11:16:09,163 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20b332191a688c1f969d80830557be9b/B of 20b332191a688c1f969d80830557be9b into 5b4bdca400a84c8aaece569d4622bd8f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:09,163 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:09,163 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., storeName=20b332191a688c1f969d80830557be9b/B, priority=13, startTime=1732187768735; duration=0sec 2024-11-21T11:16:09,164 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:16:09,164 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20b332191a688c1f969d80830557be9b:B 2024-11-21T11:16:09,164 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:16:09,164 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:16:09,164 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 20b332191a688c1f969d80830557be9b/C is initiating minor compaction (all files) 2024-11-21T11:16:09,164 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20b332191a688c1f969d80830557be9b/C in TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:09,165 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/536829c7a7f546ada70b42e40274726c, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/801589f6a695477ab917884677363dce, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/5d61fdcd11064d02a858a79487d5bf41] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp, totalSize=35.9 K 2024-11-21T11:16:09,165 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 536829c7a7f546ada70b42e40274726c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732187764965 2024-11-21T11:16:09,165 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 801589f6a695477ab917884677363dce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732187765636 2024-11-21T11:16:09,165 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20b332191a688c1f969d80830557be9b/A of 20b332191a688c1f969d80830557be9b into b1c642677b17427f96a946fba4791682(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:09,165 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:09,165 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d61fdcd11064d02a858a79487d5bf41, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732187767840 2024-11-21T11:16:09,165 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., storeName=20b332191a688c1f969d80830557be9b/A, priority=13, startTime=1732187768735; duration=0sec 2024-11-21T11:16:09,166 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:09,166 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20b332191a688c1f969d80830557be9b:A 2024-11-21T11:16:09,171 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20b332191a688c1f969d80830557be9b#C#compaction#419 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:09,171 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/a028574cd9a145c783a3e28447f6ed28 is 50, key is test_row_0/C:col10/1732187767852/Put/seqid=0 2024-11-21T11:16:09,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742324_1500 (size=12595) 2024-11-21T11:16:09,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T11:16:09,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187829215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:09,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:09,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187829215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:09,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:09,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:09,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187829215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:09,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:09,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:09,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:09,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187829225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:09,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:09,264 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/777d4b4e6cb04cf9b0e38518bac0c39e 2024-11-21T11:16:09,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/cf5dc5a62f014d49903d20ed9a90f3aa is 50, key is test_row_0/B:col10/1732187767892/Put/seqid=0 2024-11-21T11:16:09,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742325_1501 (size=12151) 2024-11-21T11:16:09,276 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/cf5dc5a62f014d49903d20ed9a90f3aa 2024-11-21T11:16:09,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/428a44e66eb747fb844f9c2559ad5313 is 50, key is test_row_0/C:col10/1732187767892/Put/seqid=0 2024-11-21T11:16:09,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742326_1502 (size=12151) 2024-11-21T11:16:09,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:09,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187829435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:09,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:09,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187829435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:09,441 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:09,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:09,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187829435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:09,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:09,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:09,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:09,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187829445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:09,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:09,580 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/a028574cd9a145c783a3e28447f6ed28 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/a028574cd9a145c783a3e28447f6ed28 2024-11-21T11:16:09,584 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20b332191a688c1f969d80830557be9b/C of 20b332191a688c1f969d80830557be9b into a028574cd9a145c783a3e28447f6ed28(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:09,584 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:09,584 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., storeName=20b332191a688c1f969d80830557be9b/C, priority=13, startTime=1732187768735; duration=0sec 2024-11-21T11:16:09,584 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:09,584 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20b332191a688c1f969d80830557be9b:C 2024-11-21T11:16:09,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T11:16:09,689 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/428a44e66eb747fb844f9c2559ad5313 2024-11-21T11:16:09,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/777d4b4e6cb04cf9b0e38518bac0c39e as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/777d4b4e6cb04cf9b0e38518bac0c39e 2024-11-21T11:16:09,697 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/777d4b4e6cb04cf9b0e38518bac0c39e, entries=150, sequenceid=205, filesize=11.9 K 2024-11-21T11:16:09,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/cf5dc5a62f014d49903d20ed9a90f3aa as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/cf5dc5a62f014d49903d20ed9a90f3aa 2024-11-21T11:16:09,700 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/cf5dc5a62f014d49903d20ed9a90f3aa, entries=150, sequenceid=205, filesize=11.9 K 2024-11-21T11:16:09,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/428a44e66eb747fb844f9c2559ad5313 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/428a44e66eb747fb844f9c2559ad5313 2024-11-21T11:16:09,704 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/428a44e66eb747fb844f9c2559ad5313, entries=150, sequenceid=205, filesize=11.9 K 2024-11-21T11:16:09,705 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 20b332191a688c1f969d80830557be9b in 851ms, sequenceid=205, compaction requested=false 2024-11-21T11:16:09,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:09,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:09,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-21T11:16:09,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-21T11:16:09,708 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-21T11:16:09,708 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1570 sec 2024-11-21T11:16:09,709 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.1630 sec 2024-11-21T11:16:09,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 20b332191a688c1f969d80830557be9b 2024-11-21T11:16:09,749 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20b332191a688c1f969d80830557be9b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-21T11:16:09,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=A 2024-11-21T11:16:09,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:09,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=B 2024-11-21T11:16:09,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:09,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=C 2024-11-21T11:16:09,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:09,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/2102570a80734bdbac2d32c4157dc60b is 50, key is test_row_0/A:col10/1732187769104/Put/seqid=0 2024-11-21T11:16:09,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742327_1503 (size=14541) 2024-11-21T11:16:09,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:09,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187829779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:09,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:09,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:09,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187829786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:09,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:09,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:09,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187829789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:09,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:09,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:09,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187829794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:09,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:09,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:09,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187829895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:09,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:09,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:09,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187829905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:09,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:09,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187829906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:09,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:09,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:09,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187829906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:09,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:09,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:09,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:10,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:10,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187830105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:10,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:10,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:10,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:10,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:10,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187830126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:10,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:10,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187830126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:10,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:10,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:10,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:10,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:10,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:10,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187830126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:10,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:10,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:10,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:10,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:10,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:10,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/2102570a80734bdbac2d32c4157dc60b 2024-11-21T11:16:10,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/5e98377b7ab24a9bb400525a5af3f347 is 50, key is test_row_0/B:col10/1732187769104/Put/seqid=0 2024-11-21T11:16:10,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742328_1504 (size=12151) 2024-11-21T11:16:10,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:10,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187830430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:10,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:10,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187830436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:10,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:10,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:10,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:10,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:10,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:10,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187830436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:10,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:10,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:10,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:10,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:10,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:10,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:10,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187830436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:10,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:10,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:10,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:10,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/5e98377b7ab24a9bb400525a5af3f347 2024-11-21T11:16:10,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/d2ea107419c84d60a0c553d03df9897b is 50, key is test_row_0/C:col10/1732187769104/Put/seqid=0 2024-11-21T11:16:10,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742329_1505 (size=12151) 2024-11-21T11:16:10,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T11:16:10,695 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:16:10,696 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:16:10,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-21T11:16:10,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T11:16:10,698 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:16:10,698 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:16:10,698 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:16:10,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T11:16:10,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-21T11:16:10,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:10,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:16:10,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:10,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:10,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:10,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:10,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:10,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187830945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:10,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:10,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:10,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187830945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:10,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:10,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:10,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:10,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187830945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:10,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:10,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187830945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:10,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/d2ea107419c84d60a0c553d03df9897b 2024-11-21T11:16:10,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/2102570a80734bdbac2d32c4157dc60b as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/2102570a80734bdbac2d32c4157dc60b 2024-11-21T11:16:10,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/2102570a80734bdbac2d32c4157dc60b, entries=200, sequenceid=221, filesize=14.2 K 2024-11-21T11:16:10,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/5e98377b7ab24a9bb400525a5af3f347 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/5e98377b7ab24a9bb400525a5af3f347 2024-11-21T11:16:10,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/5e98377b7ab24a9bb400525a5af3f347, entries=150, sequenceid=221, filesize=11.9 K 2024-11-21T11:16:10,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/d2ea107419c84d60a0c553d03df9897b as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/d2ea107419c84d60a0c553d03df9897b 2024-11-21T11:16:10,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/d2ea107419c84d60a0c553d03df9897b, entries=150, sequenceid=221, filesize=11.9 K 2024-11-21T11:16:10,995 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 20b332191a688c1f969d80830557be9b in 1245ms, sequenceid=221, compaction requested=true 2024-11-21T11:16:10,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:10,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20b332191a688c1f969d80830557be9b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:16:10,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:10,995 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:16:10,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20b332191a688c1f969d80830557be9b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:16:10,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:10,995 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:16:10,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20b332191a688c1f969d80830557be9b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:16:10,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:16:10,996 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:16:10,996 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 20b332191a688c1f969d80830557be9b/A is initiating minor compaction (all files) 2024-11-21T11:16:10,996 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:16:10,996 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20b332191a688c1f969d80830557be9b/A in TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:10,996 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 20b332191a688c1f969d80830557be9b/B is initiating minor compaction (all files) 2024-11-21T11:16:10,996 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/b1c642677b17427f96a946fba4791682, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/777d4b4e6cb04cf9b0e38518bac0c39e, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/2102570a80734bdbac2d32c4157dc60b] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp, totalSize=38.4 K 2024-11-21T11:16:10,996 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20b332191a688c1f969d80830557be9b/B in TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:10,996 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/5b4bdca400a84c8aaece569d4622bd8f, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/cf5dc5a62f014d49903d20ed9a90f3aa, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/5e98377b7ab24a9bb400525a5af3f347] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp, totalSize=36.0 K 2024-11-21T11:16:10,996 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1c642677b17427f96a946fba4791682, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732187767840 2024-11-21T11:16:10,996 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b4bdca400a84c8aaece569d4622bd8f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732187767840 2024-11-21T11:16:10,997 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 777d4b4e6cb04cf9b0e38518bac0c39e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732187767892 2024-11-21T11:16:10,997 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting cf5dc5a62f014d49903d20ed9a90f3aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732187767892 2024-11-21T11:16:10,997 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2102570a80734bdbac2d32c4157dc60b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732187769092 2024-11-21T11:16:10,997 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e98377b7ab24a9bb400525a5af3f347, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732187769103 2024-11-21T11:16:11,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-21T11:16:11,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:11,004 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 20b332191a688c1f969d80830557be9b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T11:16:11,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=A 2024-11-21T11:16:11,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:11,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=B 2024-11-21T11:16:11,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:11,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=C 2024-11-21T11:16:11,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:11,006 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20b332191a688c1f969d80830557be9b#A#compaction#425 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:11,007 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/aba528cf33554b31a9a5589debe0af0a is 50, key is test_row_0/A:col10/1732187769104/Put/seqid=0 2024-11-21T11:16:11,009 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20b332191a688c1f969d80830557be9b#B#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:11,009 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/95db0830809c4329b7b9c8aac332ecc4 is 50, key is test_row_0/B:col10/1732187769104/Put/seqid=0 2024-11-21T11:16:11,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/e3ec4ad6a93d4e2fb9668aee151c8f06 is 50, key is test_row_0/A:col10/1732187769784/Put/seqid=0 2024-11-21T11:16:11,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T11:16:11,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742331_1507 (size=12697) 2024-11-21T11:16:11,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742330_1506 (size=12697) 2024-11-21T11:16:11,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742332_1508 (size=12151) 2024-11-21T11:16:11,032 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/e3ec4ad6a93d4e2fb9668aee151c8f06 2024-11-21T11:16:11,037 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/aba528cf33554b31a9a5589debe0af0a as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/aba528cf33554b31a9a5589debe0af0a 2024-11-21T11:16:11,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/2aa3d0c257d34fd1b2763c53f52bde9b is 50, key is test_row_0/B:col10/1732187769784/Put/seqid=0 2024-11-21T11:16:11,046 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20b332191a688c1f969d80830557be9b/A of 20b332191a688c1f969d80830557be9b into aba528cf33554b31a9a5589debe0af0a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:11,046 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:11,046 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., storeName=20b332191a688c1f969d80830557be9b/A, priority=13, startTime=1732187770995; duration=0sec 2024-11-21T11:16:11,046 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:16:11,046 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20b332191a688c1f969d80830557be9b:A 2024-11-21T11:16:11,046 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:16:11,047 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:16:11,047 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 20b332191a688c1f969d80830557be9b/C is initiating minor compaction (all files) 2024-11-21T11:16:11,047 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20b332191a688c1f969d80830557be9b/C in TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:11,047 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/a028574cd9a145c783a3e28447f6ed28, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/428a44e66eb747fb844f9c2559ad5313, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/d2ea107419c84d60a0c553d03df9897b] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp, totalSize=36.0 K 2024-11-21T11:16:11,048 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting a028574cd9a145c783a3e28447f6ed28, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732187767840 2024-11-21T11:16:11,048 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 428a44e66eb747fb844f9c2559ad5313, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732187767892 2024-11-21T11:16:11,049 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2ea107419c84d60a0c553d03df9897b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732187769103 2024-11-21T11:16:11,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742333_1509 (size=12151) 2024-11-21T11:16:11,059 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20b332191a688c1f969d80830557be9b#C#compaction#429 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:11,060 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/2672f9a0ad3c42019b3bac0d496a7d26 is 50, key is test_row_0/C:col10/1732187769104/Put/seqid=0 2024-11-21T11:16:11,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742334_1510 (size=12697) 2024-11-21T11:16:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T11:16:11,424 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/95db0830809c4329b7b9c8aac332ecc4 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/95db0830809c4329b7b9c8aac332ecc4 2024-11-21T11:16:11,429 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20b332191a688c1f969d80830557be9b/B of 20b332191a688c1f969d80830557be9b into 95db0830809c4329b7b9c8aac332ecc4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:11,429 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:11,429 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., storeName=20b332191a688c1f969d80830557be9b/B, priority=13, startTime=1732187770995; duration=0sec 2024-11-21T11:16:11,429 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:11,429 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20b332191a688c1f969d80830557be9b:B 2024-11-21T11:16:11,459 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/2aa3d0c257d34fd1b2763c53f52bde9b 2024-11-21T11:16:11,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/0cd67a696d8f42f29773bc0837dd6096 is 50, key is test_row_0/C:col10/1732187769784/Put/seqid=0 2024-11-21T11:16:11,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742335_1511 (size=12151) 2024-11-21T11:16:11,470 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/2672f9a0ad3c42019b3bac0d496a7d26 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/2672f9a0ad3c42019b3bac0d496a7d26 2024-11-21T11:16:11,470 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/0cd67a696d8f42f29773bc0837dd6096 2024-11-21T11:16:11,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/e3ec4ad6a93d4e2fb9668aee151c8f06 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/e3ec4ad6a93d4e2fb9668aee151c8f06 2024-11-21T11:16:11,477 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20b332191a688c1f969d80830557be9b/C of 20b332191a688c1f969d80830557be9b into 2672f9a0ad3c42019b3bac0d496a7d26(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:11,477 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:11,477 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., storeName=20b332191a688c1f969d80830557be9b/C, priority=13, startTime=1732187770995; duration=0sec 2024-11-21T11:16:11,478 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:11,478 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20b332191a688c1f969d80830557be9b:C 2024-11-21T11:16:11,480 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/e3ec4ad6a93d4e2fb9668aee151c8f06, entries=150, sequenceid=240, filesize=11.9 K 2024-11-21T11:16:11,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/2aa3d0c257d34fd1b2763c53f52bde9b as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/2aa3d0c257d34fd1b2763c53f52bde9b 2024-11-21T11:16:11,484 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/2aa3d0c257d34fd1b2763c53f52bde9b, entries=150, sequenceid=240, filesize=11.9 K 2024-11-21T11:16:11,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/0cd67a696d8f42f29773bc0837dd6096 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/0cd67a696d8f42f29773bc0837dd6096 2024-11-21T11:16:11,489 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/0cd67a696d8f42f29773bc0837dd6096, entries=150, sequenceid=240, filesize=11.9 K 2024-11-21T11:16:11,490 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 20b332191a688c1f969d80830557be9b in 485ms, sequenceid=240, compaction requested=false 2024-11-21T11:16:11,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:11,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:11,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-21T11:16:11,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-21T11:16:11,494 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-21T11:16:11,494 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 793 msec 2024-11-21T11:16:11,495 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 798 msec 2024-11-21T11:16:11,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T11:16:11,835 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:16:11,836 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:16:11,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-21T11:16:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T11:16:11,838 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:16:11,839 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:16:11,839 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:16:11,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T11:16:11,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 20b332191a688c1f969d80830557be9b 2024-11-21T11:16:11,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20b332191a688c1f969d80830557be9b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T11:16:11,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=A 2024-11-21T11:16:11,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:11,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=B 2024-11-21T11:16:11,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:11,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=C 2024-11-21T11:16:11,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:11,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-21T11:16:11,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:11,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:16:11,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:11,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:11,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:12,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/48e6852edaef446f85c95c0082f68dc7 is 50, key is test_row_0/A:col10/1732187771973/Put/seqid=0 2024-11-21T11:16:12,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742336_1512 (size=12151) 2024-11-21T11:16:12,060 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732187772017, max timestamp=1732187772056Num uniques 859; 2024-11-21T11:16:12,060 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732187772017, max timestamp=1732187772056Num uniques 859; 2024-11-21T11:16:12,060 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732187772017, max timestamp=1732187772056Num uniques 859; 2024-11-21T11:16:12,060 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T11:16:12,060 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T11:16:12,060 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T11:16:12,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:12,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187832091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:12,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:12,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:12,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:12,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:12,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187832092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:12,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:12,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187832098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:12,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:12,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187832104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:12,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:12,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:12,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:12,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-21T11:16:12,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:12,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:16:12,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:12,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:12,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T11:16:12,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:12,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187832215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:12,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:12,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:12,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:12,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187832216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:12,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:12,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:12,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:12,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187832216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:12,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:12,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:12,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:12,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187832216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:12,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-21T11:16:12,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:12,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:16:12,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:12,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:12,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:12,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/48e6852edaef446f85c95c0082f68dc7 2024-11-21T11:16:12,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/c0d47974745d45baa7f510327d2a61ef is 50, key is test_row_0/B:col10/1732187771973/Put/seqid=0 2024-11-21T11:16:12,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742337_1513 (size=12151) 2024-11-21T11:16:12,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:12,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187832435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:12,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:12,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:12,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187832436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:12,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:12,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:12,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:12,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:12,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:12,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187832436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:12,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:12,447 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:12,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:12,447 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:12,447 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:12,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187832436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:12,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:12,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:12,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:12,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-21T11:16:12,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:12,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:16:12,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:12,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:12,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:12,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T11:16:12,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-21T11:16:12,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:12,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:16:12,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:12,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:12,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:12,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:12,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-21T11:16:12,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:12,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:16:12,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:12,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:12,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:12,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:12,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:12,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187832755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:12,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:12,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:12,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:12,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187832756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:12,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:12,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:12,761 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:12,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187832756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:12,761 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:12,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:12,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:12,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:12,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:12,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:12,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187832756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:12,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:12,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:12,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:12,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/c0d47974745d45baa7f510327d2a61ef 2024-11-21T11:16:12,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/d1b136a85c61454ea8edb92770b03cdd is 50, key is test_row_0/C:col10/1732187771973/Put/seqid=0 2024-11-21T11:16:12,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742338_1514 (size=12151) 2024-11-21T11:16:12,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-21T11:16:12,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:12,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:16:12,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:12,910 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:12,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:12,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:12,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T11:16:13,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-21T11:16:13,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:13,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:16:13,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:13,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:13,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:13,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:13,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-21T11:16:13,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:13,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:16:13,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:13,218 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:13,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:13,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:13,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/d1b136a85c61454ea8edb92770b03cdd 2024-11-21T11:16:13,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/48e6852edaef446f85c95c0082f68dc7 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/48e6852edaef446f85c95c0082f68dc7 2024-11-21T11:16:13,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:13,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187833265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:13,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:13,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:13,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:13,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/48e6852edaef446f85c95c0082f68dc7, entries=150, sequenceid=253, filesize=11.9 K 2024-11-21T11:16:13,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/c0d47974745d45baa7f510327d2a61ef as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/c0d47974745d45baa7f510327d2a61ef 2024-11-21T11:16:13,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/c0d47974745d45baa7f510327d2a61ef, entries=150, sequenceid=253, filesize=11.9 K 2024-11-21T11:16:13,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:13,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187833276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:13,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:13,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187833276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:13,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:13,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:13,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:13,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:13,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187833276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:13,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/d1b136a85c61454ea8edb92770b03cdd as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/d1b136a85c61454ea8edb92770b03cdd 2024-11-21T11:16:13,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:13,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:13,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:13,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:13,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:13,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:13,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/d1b136a85c61454ea8edb92770b03cdd, entries=150, sequenceid=253, filesize=11.9 K 2024-11-21T11:16:13,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 20b332191a688c1f969d80830557be9b in 1319ms, sequenceid=253, compaction requested=true 2024-11-21T11:16:13,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:13,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20b332191a688c1f969d80830557be9b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:16:13,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:13,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20b332191a688c1f969d80830557be9b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:16:13,293 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:16:13,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:16:13,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20b332191a688c1f969d80830557be9b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:16:13,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T11:16:13,293 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:16:13,295 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:16:13,295 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 20b332191a688c1f969d80830557be9b/B is initiating minor compaction (all files) 2024-11-21T11:16:13,295 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20b332191a688c1f969d80830557be9b/B in TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:13,295 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/95db0830809c4329b7b9c8aac332ecc4, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/2aa3d0c257d34fd1b2763c53f52bde9b, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/c0d47974745d45baa7f510327d2a61ef] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp, totalSize=36.1 K 2024-11-21T11:16:13,295 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 95db0830809c4329b7b9c8aac332ecc4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732187769103 2024-11-21T11:16:13,296 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 2aa3d0c257d34fd1b2763c53f52bde9b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732187769778 2024-11-21T11:16:13,296 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting c0d47974745d45baa7f510327d2a61ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732187771968 2024-11-21T11:16:13,297 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:16:13,297 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 20b332191a688c1f969d80830557be9b/A is initiating minor compaction (all files) 2024-11-21T11:16:13,297 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20b332191a688c1f969d80830557be9b/A in TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:13,297 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/aba528cf33554b31a9a5589debe0af0a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/e3ec4ad6a93d4e2fb9668aee151c8f06, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/48e6852edaef446f85c95c0082f68dc7] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp, totalSize=36.1 K 2024-11-21T11:16:13,297 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting aba528cf33554b31a9a5589debe0af0a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732187769103 2024-11-21T11:16:13,298 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3ec4ad6a93d4e2fb9668aee151c8f06, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732187769778 2024-11-21T11:16:13,299 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48e6852edaef446f85c95c0082f68dc7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732187771968 2024-11-21T11:16:13,312 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20b332191a688c1f969d80830557be9b#B#compaction#434 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:13,313 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/4ddf85f57d6243658d05faf91ce0284a is 50, key is test_row_0/B:col10/1732187771973/Put/seqid=0 2024-11-21T11:16:13,319 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20b332191a688c1f969d80830557be9b#A#compaction#435 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:13,320 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/2892e91db57b4ded99cffa634cb767f9 is 50, key is test_row_0/A:col10/1732187771973/Put/seqid=0 2024-11-21T11:16:13,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742339_1515 (size=12799) 2024-11-21T11:16:13,356 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/4ddf85f57d6243658d05faf91ce0284a as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/4ddf85f57d6243658d05faf91ce0284a 2024-11-21T11:16:13,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742340_1516 (size=12799) 2024-11-21T11:16:13,361 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20b332191a688c1f969d80830557be9b/B of 20b332191a688c1f969d80830557be9b into 4ddf85f57d6243658d05faf91ce0284a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:13,362 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:13,362 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., storeName=20b332191a688c1f969d80830557be9b/B, priority=13, startTime=1732187773293; duration=0sec 2024-11-21T11:16:13,362 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:16:13,362 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20b332191a688c1f969d80830557be9b:B 2024-11-21T11:16:13,362 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:16:13,363 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:16:13,363 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 20b332191a688c1f969d80830557be9b/C is initiating minor compaction (all files) 2024-11-21T11:16:13,363 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20b332191a688c1f969d80830557be9b/C in TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:13,363 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/2672f9a0ad3c42019b3bac0d496a7d26, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/0cd67a696d8f42f29773bc0837dd6096, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/d1b136a85c61454ea8edb92770b03cdd] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp, totalSize=36.1 K 2024-11-21T11:16:13,363 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 2672f9a0ad3c42019b3bac0d496a7d26, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732187769103 2024-11-21T11:16:13,364 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cd67a696d8f42f29773bc0837dd6096, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732187769778 2024-11-21T11:16:13,364 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting d1b136a85c61454ea8edb92770b03cdd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732187771968 2024-11-21T11:16:13,367 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/2892e91db57b4ded99cffa634cb767f9 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/2892e91db57b4ded99cffa634cb767f9 2024-11-21T11:16:13,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-21T11:16:13,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:13,372 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 20b332191a688c1f969d80830557be9b 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-21T11:16:13,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=A 2024-11-21T11:16:13,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:16:13,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=B 2024-11-21T11:16:13,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:16:13,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=C 2024-11-21T11:16:13,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:16:13,373 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20b332191a688c1f969d80830557be9b/A of 20b332191a688c1f969d80830557be9b into 2892e91db57b4ded99cffa634cb767f9(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:13,373 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:13,374 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., storeName=20b332191a688c1f969d80830557be9b/A, priority=13, startTime=1732187773293; duration=0sec 2024-11-21T11:16:13,374 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:13,374 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20b332191a688c1f969d80830557be9b:A 2024-11-21T11:16:13,378 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20b332191a688c1f969d80830557be9b#C#compaction#436 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:13,379 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/51d981dd86d7479f9a68cef3d703afe7 is 50, key is test_row_0/C:col10/1732187771973/Put/seqid=0 2024-11-21T11:16:13,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/12c6bfe70fc44db9b1ed8f801b67fe72 is 50, key is test_row_0/A:col10/1732187772103/Put/seqid=0 2024-11-21T11:16:13,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742341_1517 (size=12799) 2024-11-21T11:16:13,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742342_1518 (size=12301) 2024-11-21T11:16:13,403 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/12c6bfe70fc44db9b1ed8f801b67fe72 2024-11-21T11:16:13,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/b00caab6b24b49248b5fcf32a72e10de is 50, key is test_row_0/B:col10/1732187772103/Put/seqid=0 2024-11-21T11:16:13,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742343_1519 (size=12301) 2024-11-21T11:16:13,424 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/b00caab6b24b49248b5fcf32a72e10de 2024-11-21T11:16:13,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/33b7f879b2aa464bb26e2774a45b8447 is 50, key is test_row_0/C:col10/1732187772103/Put/seqid=0 2024-11-21T11:16:13,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742344_1520 (size=12301) 2024-11-21T11:16:13,438 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/33b7f879b2aa464bb26e2774a45b8447 2024-11-21T11:16:13,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/12c6bfe70fc44db9b1ed8f801b67fe72 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/12c6bfe70fc44db9b1ed8f801b67fe72 2024-11-21T11:16:13,454 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/12c6bfe70fc44db9b1ed8f801b67fe72, entries=150, sequenceid=290, filesize=12.0 K 2024-11-21T11:16:13,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/b00caab6b24b49248b5fcf32a72e10de as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/b00caab6b24b49248b5fcf32a72e10de 2024-11-21T11:16:13,460 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/b00caab6b24b49248b5fcf32a72e10de, entries=150, sequenceid=290, filesize=12.0 K 2024-11-21T11:16:13,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/33b7f879b2aa464bb26e2774a45b8447 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/33b7f879b2aa464bb26e2774a45b8447 2024-11-21T11:16:13,469 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/33b7f879b2aa464bb26e2774a45b8447, entries=150, sequenceid=290, filesize=12.0 K 2024-11-21T11:16:13,470 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=0 B/0 for 20b332191a688c1f969d80830557be9b in 99ms, sequenceid=290, compaction requested=false 2024-11-21T11:16:13,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:13,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:13,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-21T11:16:13,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-21T11:16:13,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-21T11:16:13,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6330 sec 2024-11-21T11:16:13,479 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.6410 sec 2024-11-21T11:16:13,809 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/51d981dd86d7479f9a68cef3d703afe7 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/51d981dd86d7479f9a68cef3d703afe7 2024-11-21T11:16:13,816 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20b332191a688c1f969d80830557be9b/C of 20b332191a688c1f969d80830557be9b into 51d981dd86d7479f9a68cef3d703afe7(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:13,816 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:13,816 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., storeName=20b332191a688c1f969d80830557be9b/C, priority=13, startTime=1732187773293; duration=0sec 2024-11-21T11:16:13,816 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:13,816 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20b332191a688c1f969d80830557be9b:C 2024-11-21T11:16:13,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T11:16:13,985 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:16:13,986 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:16:13,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-21T11:16:13,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T11:16:13,989 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:16:13,990 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:16:13,990 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:16:14,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T11:16:14,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-21T11:16:14,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:14,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:14,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:14,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-21T11:16:14,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-21T11:16:14,148 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-21T11:16:14,149 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 156 msec 2024-11-21T11:16:14,150 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 163 msec 2024-11-21T11:16:14,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 20b332191a688c1f969d80830557be9b 2024-11-21T11:16:14,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20b332191a688c1f969d80830557be9b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T11:16:14,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=A 2024-11-21T11:16:14,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:14,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=B 2024-11-21T11:16:14,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:14,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=C 2024-11-21T11:16:14,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:14,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T11:16:14,306 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:16:14,306 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:16:14,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-21T11:16:14,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/792acd62b35b472dbcd2d6b9cafa95e8 is 50, key is test_row_0/A:col10/1732187774301/Put/seqid=0 2024-11-21T11:16:14,308 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:16:14,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-21T11:16:14,309 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:16:14,309 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:16:14,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742345_1521 (size=12301) 2024-11-21T11:16:14,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732187774347, max timestamp=1732187774371Num uniques 859; 2024-11-21T11:16:14,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T11:16:14,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732187774347, max timestamp=1732187774371Num uniques 859; 2024-11-21T11:16:14,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T11:16:14,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732187774347, max timestamp=1732187774371Num uniques 859; 2024-11-21T11:16:14,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T11:16:14,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-21T11:16:14,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:14,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187834408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:14,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:14,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:14,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187834422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:14,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:14,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:14,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:14,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:14,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:14,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:14,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187834423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:14,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:14,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:14,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:14,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:14,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187834434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:14,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:14,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:14,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:14,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-21T11:16:14,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:14,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:16:14,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:14,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:14,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:14,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:14,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:14,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187834545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:14,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:14,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:14,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:14,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:14,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187834545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:14,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:14,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:14,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187834545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:14,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:14,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:14,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:14,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:14,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:14,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:14,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187834556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:14,567 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:14,567 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:14,567 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:14,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-21T11:16:14,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:14,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:16:14,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:14,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:14,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:14,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:14,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-21T11:16:14,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/792acd62b35b472dbcd2d6b9cafa95e8 2024-11-21T11:16:14,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/ff4882fb22064b158f466fef1b63fa88 is 50, key is test_row_0/B:col10/1732187774301/Put/seqid=0 2024-11-21T11:16:14,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742346_1522 (size=12301) 2024-11-21T11:16:14,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-21T11:16:14,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:14,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:16:14,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:14,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:14,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:14,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:14,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:14,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187834765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:14,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:14,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187834765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:14,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:14,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:14,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187834765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:14,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:14,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:14,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:14,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:14,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:14,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:14,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187834775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:14,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:14,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:14,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:14,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-21T11:16:14,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:14,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:16:14,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:14,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:14,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:14,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:14,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-21T11:16:15,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-21T11:16:15,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:15,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:16:15,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:15,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:15,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:15,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:15,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:15,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187835085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:15,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:15,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:15,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:15,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187835085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:15,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:15,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:15,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:15,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:15,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187835086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:15,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:15,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:15,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:15,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:15,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:15,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187835095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:15,110 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:15,110 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:15,110 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:15,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/ff4882fb22064b158f466fef1b63fa88 2024-11-21T11:16:15,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/3c2afef689794769bc8bc148f223cc9d is 50, key is test_row_0/C:col10/1732187774301/Put/seqid=0 2024-11-21T11:16:15,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742347_1523 (size=12301) 2024-11-21T11:16:15,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/3c2afef689794769bc8bc148f223cc9d 2024-11-21T11:16:15,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/792acd62b35b472dbcd2d6b9cafa95e8 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/792acd62b35b472dbcd2d6b9cafa95e8 2024-11-21T11:16:15,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/792acd62b35b472dbcd2d6b9cafa95e8, entries=150, sequenceid=303, filesize=12.0 K 2024-11-21T11:16:15,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/ff4882fb22064b158f466fef1b63fa88 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/ff4882fb22064b158f466fef1b63fa88 2024-11-21T11:16:15,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/ff4882fb22064b158f466fef1b63fa88, entries=150, sequenceid=303, filesize=12.0 K 2024-11-21T11:16:15,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/3c2afef689794769bc8bc148f223cc9d as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/3c2afef689794769bc8bc148f223cc9d 2024-11-21T11:16:15,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/3c2afef689794769bc8bc148f223cc9d, entries=150, sequenceid=303, filesize=12.0 K 2024-11-21T11:16:15,172 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 20b332191a688c1f969d80830557be9b in 869ms, sequenceid=303, compaction requested=true 2024-11-21T11:16:15,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:15,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20b332191a688c1f969d80830557be9b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:16:15,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:15,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20b332191a688c1f969d80830557be9b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:16:15,172 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:16:15,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:16:15,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20b332191a688c1f969d80830557be9b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:16:15,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T11:16:15,173 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:16:15,173 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 20b332191a688c1f969d80830557be9b/A is initiating minor compaction (all files) 2024-11-21T11:16:15,173 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20b332191a688c1f969d80830557be9b/A in TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:15,173 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/2892e91db57b4ded99cffa634cb767f9, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/12c6bfe70fc44db9b1ed8f801b67fe72, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/792acd62b35b472dbcd2d6b9cafa95e8] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp, totalSize=36.5 K 2024-11-21T11:16:15,174 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:16:15,174 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2892e91db57b4ded99cffa634cb767f9, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732187771968 2024-11-21T11:16:15,174 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12c6bfe70fc44db9b1ed8f801b67fe72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732187772076 2024-11-21T11:16:15,174 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 792acd62b35b472dbcd2d6b9cafa95e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732187774300 2024-11-21T11:16:15,175 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:16:15,175 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 20b332191a688c1f969d80830557be9b/B is initiating minor compaction (all files) 2024-11-21T11:16:15,175 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20b332191a688c1f969d80830557be9b/B in TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:15,175 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/4ddf85f57d6243658d05faf91ce0284a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/b00caab6b24b49248b5fcf32a72e10de, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/ff4882fb22064b158f466fef1b63fa88] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp, totalSize=36.5 K 2024-11-21T11:16:15,175 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ddf85f57d6243658d05faf91ce0284a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732187771968 2024-11-21T11:16:15,176 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting b00caab6b24b49248b5fcf32a72e10de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732187772076 2024-11-21T11:16:15,176 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting ff4882fb22064b158f466fef1b63fa88, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732187774300 2024-11-21T11:16:15,185 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20b332191a688c1f969d80830557be9b#A#compaction#443 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:15,185 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/4579fe9d3ea74633ae07f99e0f6cec42 is 50, key is test_row_0/A:col10/1732187774301/Put/seqid=0 2024-11-21T11:16:15,188 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20b332191a688c1f969d80830557be9b#B#compaction#444 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:15,189 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/09c3c617c04a4abbb6dc222319a62b74 is 50, key is test_row_0/B:col10/1732187774301/Put/seqid=0 2024-11-21T11:16:15,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742348_1524 (size=13051) 2024-11-21T11:16:15,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742349_1525 (size=13051) 2024-11-21T11:16:15,209 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/09c3c617c04a4abbb6dc222319a62b74 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/09c3c617c04a4abbb6dc222319a62b74 2024-11-21T11:16:15,213 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20b332191a688c1f969d80830557be9b/B of 20b332191a688c1f969d80830557be9b into 09c3c617c04a4abbb6dc222319a62b74(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:15,213 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:15,213 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., storeName=20b332191a688c1f969d80830557be9b/B, priority=13, startTime=1732187775172; duration=0sec 2024-11-21T11:16:15,213 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:16:15,214 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20b332191a688c1f969d80830557be9b:B 2024-11-21T11:16:15,214 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:16:15,214 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:16:15,215 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 20b332191a688c1f969d80830557be9b/C is initiating minor compaction (all files) 2024-11-21T11:16:15,215 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20b332191a688c1f969d80830557be9b/C in TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:15,215 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/51d981dd86d7479f9a68cef3d703afe7, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/33b7f879b2aa464bb26e2774a45b8447, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/3c2afef689794769bc8bc148f223cc9d] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp, totalSize=36.5 K 2024-11-21T11:16:15,215 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 51d981dd86d7479f9a68cef3d703afe7, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732187771968 2024-11-21T11:16:15,215 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 33b7f879b2aa464bb26e2774a45b8447, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732187772076 2024-11-21T11:16:15,216 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c2afef689794769bc8bc148f223cc9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732187774300 2024-11-21T11:16:15,221 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20b332191a688c1f969d80830557be9b#C#compaction#445 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:15,222 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/cd19d323871d465e8cb998bff320082c is 50, key is test_row_0/C:col10/1732187774301/Put/seqid=0 2024-11-21T11:16:15,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-21T11:16:15,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:15,229 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 20b332191a688c1f969d80830557be9b 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-21T11:16:15,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=A 2024-11-21T11:16:15,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:16:15,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=B 2024-11-21T11:16:15,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:16:15,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=C 2024-11-21T11:16:15,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:16:15,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/8703b44b876448e892f5a0b20249c3d7 is 50, key is test_row_0/A:col10/1732187774420/Put/seqid=0 2024-11-21T11:16:15,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742350_1526 (size=13051) 2024-11-21T11:16:15,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742351_1527 (size=12301) 2024-11-21T11:16:15,263 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/8703b44b876448e892f5a0b20249c3d7 2024-11-21T11:16:15,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/bdf18b8d175948b4b3ea6c8e3879190a is 50, key is test_row_0/B:col10/1732187774420/Put/seqid=0 2024-11-21T11:16:15,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742352_1528 (size=12301) 2024-11-21T11:16:15,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-21T11:16:15,607 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/4579fe9d3ea74633ae07f99e0f6cec42 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/4579fe9d3ea74633ae07f99e0f6cec42 2024-11-21T11:16:15,613 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20b332191a688c1f969d80830557be9b/A of 20b332191a688c1f969d80830557be9b into 4579fe9d3ea74633ae07f99e0f6cec42(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:15,614 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:15,614 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., storeName=20b332191a688c1f969d80830557be9b/A, priority=13, startTime=1732187775172; duration=0sec 2024-11-21T11:16:15,614 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:15,614 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20b332191a688c1f969d80830557be9b:A 2024-11-21T11:16:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 20b332191a688c1f969d80830557be9b 2024-11-21T11:16:15,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:16:15,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:15,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187835634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:15,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:15,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:15,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:15,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:15,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187835645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:15,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:15,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:15,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:15,656 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/cd19d323871d465e8cb998bff320082c as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/cd19d323871d465e8cb998bff320082c 2024-11-21T11:16:15,661 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20b332191a688c1f969d80830557be9b/C of 20b332191a688c1f969d80830557be9b into cd19d323871d465e8cb998bff320082c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:15,661 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:15,661 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., storeName=20b332191a688c1f969d80830557be9b/C, priority=13, startTime=1732187775172; duration=0sec 2024-11-21T11:16:15,661 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:15,661 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20b332191a688c1f969d80830557be9b:C 2024-11-21T11:16:15,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:15,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187835647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:15,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:15,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187835647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:15,697 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/bdf18b8d175948b4b3ea6c8e3879190a 2024-11-21T11:16:15,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/ab892f0ef9e4401c9873751e6bdfd900 is 50, key is test_row_0/C:col10/1732187774420/Put/seqid=0 2024-11-21T11:16:15,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742353_1529 (size=12301) 2024-11-21T11:16:15,711 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/ab892f0ef9e4401c9873751e6bdfd900 2024-11-21T11:16:15,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/8703b44b876448e892f5a0b20249c3d7 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/8703b44b876448e892f5a0b20249c3d7 2024-11-21T11:16:15,727 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/8703b44b876448e892f5a0b20249c3d7, entries=150, sequenceid=339, filesize=12.0 K 2024-11-21T11:16:15,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/bdf18b8d175948b4b3ea6c8e3879190a as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/bdf18b8d175948b4b3ea6c8e3879190a 2024-11-21T11:16:15,731 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/bdf18b8d175948b4b3ea6c8e3879190a, entries=150, sequenceid=339, filesize=12.0 K 2024-11-21T11:16:15,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/ab892f0ef9e4401c9873751e6bdfd900 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/ab892f0ef9e4401c9873751e6bdfd900 2024-11-21T11:16:15,735 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/ab892f0ef9e4401c9873751e6bdfd900, entries=150, sequenceid=339, filesize=12.0 K 2024-11-21T11:16:15,738 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=53.67 KB/54960 for 20b332191a688c1f969d80830557be9b in 509ms, sequenceid=339, compaction requested=false 2024-11-21T11:16:15,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:15,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:15,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-21T11:16:15,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-21T11:16:15,752 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-21T11:16:15,752 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4310 sec 2024-11-21T11:16:15,754 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.4470 sec 2024-11-21T11:16:15,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 20b332191a688c1f969d80830557be9b 2024-11-21T11:16:15,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20b332191a688c1f969d80830557be9b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-21T11:16:15,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=A 2024-11-21T11:16:15,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:15,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=B 2024-11-21T11:16:15,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:15,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=C 2024-11-21T11:16:15,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:15,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/edb9262a57364ee2b940919baa6864ec is 50, key is test_row_0/A:col10/1732187775644/Put/seqid=0 2024-11-21T11:16:15,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742354_1530 (size=12297) 2024-11-21T11:16:15,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/edb9262a57364ee2b940919baa6864ec 2024-11-21T11:16:15,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/94574e1a1c50449498ce268e1c032f46 is 50, key is test_row_0/B:col10/1732187775644/Put/seqid=0 2024-11-21T11:16:15,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:15,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187835814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:15,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:15,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:15,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:15,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:15,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187835824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:15,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:15,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:15,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:15,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742355_1531 (size=9857) 2024-11-21T11:16:15,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:15,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187835827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:15,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:15,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187835827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:15,840 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:15,840 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:15,840 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:15,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:15,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:15,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:15,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:15,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187835936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:15,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:15,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:15,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:15,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187835938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:15,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:15,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:15,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:15,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:15,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:15,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187835945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:15,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:15,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:15,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187835945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:15,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:15,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:15,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:15,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:15,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:16,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:16,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187836156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:16,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:16,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:16,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187836156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:16,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:16,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:16,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:16,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:16,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:16,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:16,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187836156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:16,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:16,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:16,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:16,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187836156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:16,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:16,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:16,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:16,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:16,232 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/94574e1a1c50449498ce268e1c032f46 2024-11-21T11:16:16,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/b681d23f1e36423cbbd5df2d07dee372 is 50, key is test_row_0/C:col10/1732187775644/Put/seqid=0 2024-11-21T11:16:16,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742356_1532 (size=9857) 2024-11-21T11:16:16,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-21T11:16:16,455 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:16:16,456 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:16:16,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-21T11:16:16,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-21T11:16:16,458 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:16:16,458 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:16:16,458 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:16:16,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:16,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187836465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:16,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:16,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187836465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:16,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:16,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:16,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:16,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:16,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187836465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:16,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:16,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:16,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:16,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:16,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:16,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187836466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:16,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:16,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:16,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:16,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:16,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:16,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-21T11:16:16,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-21T11:16:16,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:16,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:16:16,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:16,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:16,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:16,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:16,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/b681d23f1e36423cbbd5df2d07dee372 2024-11-21T11:16:16,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/edb9262a57364ee2b940919baa6864ec as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/edb9262a57364ee2b940919baa6864ec 2024-11-21T11:16:16,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/edb9262a57364ee2b940919baa6864ec, entries=150, sequenceid=354, filesize=12.0 K 2024-11-21T11:16:16,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/94574e1a1c50449498ce268e1c032f46 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/94574e1a1c50449498ce268e1c032f46 2024-11-21T11:16:16,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/94574e1a1c50449498ce268e1c032f46, entries=100, sequenceid=354, filesize=9.6 K 2024-11-21T11:16:16,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/b681d23f1e36423cbbd5df2d07dee372 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/b681d23f1e36423cbbd5df2d07dee372 2024-11-21T11:16:16,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/b681d23f1e36423cbbd5df2d07dee372, entries=100, sequenceid=354, filesize=9.6 K 2024-11-21T11:16:16,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 20b332191a688c1f969d80830557be9b in 909ms, sequenceid=354, compaction requested=true 2024-11-21T11:16:16,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:16,669 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:16:16,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20b332191a688c1f969d80830557be9b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:16:16,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:16,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20b332191a688c1f969d80830557be9b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:16:16,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:16,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20b332191a688c1f969d80830557be9b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:16:16,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:16:16,669 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:16:16,671 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37649 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:16:16,671 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 20b332191a688c1f969d80830557be9b/A is initiating minor compaction (all files) 2024-11-21T11:16:16,671 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20b332191a688c1f969d80830557be9b/A in TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:16,671 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:16:16,671 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 20b332191a688c1f969d80830557be9b/B is initiating minor compaction (all files) 2024-11-21T11:16:16,671 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/4579fe9d3ea74633ae07f99e0f6cec42, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/8703b44b876448e892f5a0b20249c3d7, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/edb9262a57364ee2b940919baa6864ec] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp, totalSize=36.8 K 2024-11-21T11:16:16,671 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20b332191a688c1f969d80830557be9b/B in TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:16,671 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/09c3c617c04a4abbb6dc222319a62b74, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/bdf18b8d175948b4b3ea6c8e3879190a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/94574e1a1c50449498ce268e1c032f46] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp, totalSize=34.4 K 2024-11-21T11:16:16,672 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4579fe9d3ea74633ae07f99e0f6cec42, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732187774300 2024-11-21T11:16:16,672 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 09c3c617c04a4abbb6dc222319a62b74, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732187774300 2024-11-21T11:16:16,672 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8703b44b876448e892f5a0b20249c3d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732187774420 2024-11-21T11:16:16,672 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting bdf18b8d175948b4b3ea6c8e3879190a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732187774420 2024-11-21T11:16:16,672 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting edb9262a57364ee2b940919baa6864ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1732187775644 2024-11-21T11:16:16,672 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 94574e1a1c50449498ce268e1c032f46, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1732187775644 2024-11-21T11:16:16,682 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20b332191a688c1f969d80830557be9b#B#compaction#452 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:16,683 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/722bd9f86f42442bbac8ff7ce7390268 is 50, key is test_row_0/B:col10/1732187775644/Put/seqid=0 2024-11-21T11:16:16,683 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20b332191a688c1f969d80830557be9b#A#compaction#453 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:16,683 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/c2ef7e9e39d04256816c0d2d7d422304 is 50, key is test_row_0/A:col10/1732187775644/Put/seqid=0 2024-11-21T11:16:16,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742357_1533 (size=13153) 2024-11-21T11:16:16,697 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/722bd9f86f42442bbac8ff7ce7390268 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/722bd9f86f42442bbac8ff7ce7390268 2024-11-21T11:16:16,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742358_1534 (size=13153) 2024-11-21T11:16:16,702 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20b332191a688c1f969d80830557be9b/B of 20b332191a688c1f969d80830557be9b into 722bd9f86f42442bbac8ff7ce7390268(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:16,702 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:16,702 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., storeName=20b332191a688c1f969d80830557be9b/B, priority=13, startTime=1732187776669; duration=0sec 2024-11-21T11:16:16,702 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:16:16,702 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20b332191a688c1f969d80830557be9b:B 2024-11-21T11:16:16,702 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:16:16,703 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:16:16,703 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 20b332191a688c1f969d80830557be9b/C is initiating minor compaction (all files) 2024-11-21T11:16:16,703 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20b332191a688c1f969d80830557be9b/C in TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:16,703 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/cd19d323871d465e8cb998bff320082c, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/ab892f0ef9e4401c9873751e6bdfd900, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/b681d23f1e36423cbbd5df2d07dee372] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp, totalSize=34.4 K 2024-11-21T11:16:16,703 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting cd19d323871d465e8cb998bff320082c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732187774300 2024-11-21T11:16:16,704 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting ab892f0ef9e4401c9873751e6bdfd900, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732187774420 2024-11-21T11:16:16,704 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting b681d23f1e36423cbbd5df2d07dee372, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1732187775644 2024-11-21T11:16:16,710 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20b332191a688c1f969d80830557be9b#C#compaction#454 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:16,711 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/8c90aecf6ea440a7b3f020cd99d8d4ab is 50, key is test_row_0/C:col10/1732187775644/Put/seqid=0 2024-11-21T11:16:16,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742359_1535 (size=13153) 2024-11-21T11:16:16,728 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/8c90aecf6ea440a7b3f020cd99d8d4ab as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/8c90aecf6ea440a7b3f020cd99d8d4ab 2024-11-21T11:16:16,732 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20b332191a688c1f969d80830557be9b/C of 20b332191a688c1f969d80830557be9b into 8c90aecf6ea440a7b3f020cd99d8d4ab(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:16,732 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:16,732 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., storeName=20b332191a688c1f969d80830557be9b/C, priority=13, startTime=1732187776669; duration=0sec 2024-11-21T11:16:16,732 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:16,732 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20b332191a688c1f969d80830557be9b:C 2024-11-21T11:16:16,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-21T11:16:16,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:16,765 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 20b332191a688c1f969d80830557be9b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T11:16:16,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=A 2024-11-21T11:16:16,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:16,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=B 2024-11-21T11:16:16,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:16,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=C 2024-11-21T11:16:16,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:16,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/f0368482359540b8ac22e9ed783b33ea is 50, key is test_row_0/A:col10/1732187775808/Put/seqid=0 2024-11-21T11:16:16,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-21T11:16:16,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742360_1536 (size=12301) 2024-11-21T11:16:16,781 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/f0368482359540b8ac22e9ed783b33ea 2024-11-21T11:16:16,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/364b3d16ddeb4c86a81ae6e428cbcfa9 is 50, key is test_row_0/B:col10/1732187775808/Put/seqid=0 2024-11-21T11:16:16,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742361_1537 (size=12301) 2024-11-21T11:16:16,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 20b332191a688c1f969d80830557be9b 2024-11-21T11:16:16,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. as already flushing 2024-11-21T11:16:17,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:17,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187837000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:17,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:17,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187837004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:17,011 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:17,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:17,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187837004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:17,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:17,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:17,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:17,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187837005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:17,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:17,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-21T11:16:17,102 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/c2ef7e9e39d04256816c0d2d7d422304 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/c2ef7e9e39d04256816c0d2d7d422304 2024-11-21T11:16:17,106 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20b332191a688c1f969d80830557be9b/A of 20b332191a688c1f969d80830557be9b into c2ef7e9e39d04256816c0d2d7d422304(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:17,106 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:17,106 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., storeName=20b332191a688c1f969d80830557be9b/A, priority=13, startTime=1732187776669; duration=0sec 2024-11-21T11:16:17,106 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:17,106 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20b332191a688c1f969d80830557be9b:A 2024-11-21T11:16:17,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:17,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187837115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:17,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:17,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187837115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:17,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:17,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:17,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187837115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:17,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:17,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:17,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:17,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187837126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:17,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:17,195 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/364b3d16ddeb4c86a81ae6e428cbcfa9 2024-11-21T11:16:17,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/2a492560610e4dd4945892f641cde478 is 50, key is test_row_0/C:col10/1732187775808/Put/seqid=0 2024-11-21T11:16:17,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742362_1538 (size=12301) 2024-11-21T11:16:17,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:17,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187837335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:17,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:17,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187837335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:17,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:17,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187837335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:17,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187837336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:17,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-21T11:16:17,607 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/2a492560610e4dd4945892f641cde478 2024-11-21T11:16:17,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/f0368482359540b8ac22e9ed783b33ea as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/f0368482359540b8ac22e9ed783b33ea 2024-11-21T11:16:17,615 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/f0368482359540b8ac22e9ed783b33ea, entries=150, sequenceid=375, filesize=12.0 K 2024-11-21T11:16:17,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/364b3d16ddeb4c86a81ae6e428cbcfa9 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/364b3d16ddeb4c86a81ae6e428cbcfa9 2024-11-21T11:16:17,619 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/364b3d16ddeb4c86a81ae6e428cbcfa9, entries=150, sequenceid=375, filesize=12.0 K 2024-11-21T11:16:17,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/2a492560610e4dd4945892f641cde478 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/2a492560610e4dd4945892f641cde478 2024-11-21T11:16:17,623 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/2a492560610e4dd4945892f641cde478, entries=150, sequenceid=375, filesize=12.0 K 2024-11-21T11:16:17,623 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 20b332191a688c1f969d80830557be9b in 858ms, sequenceid=375, compaction requested=false 2024-11-21T11:16:17,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:17,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:17,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-21T11:16:17,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-21T11:16:17,626 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-21T11:16:17,627 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1670 sec 2024-11-21T11:16:17,628 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 1.1710 sec 2024-11-21T11:16:17,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 20b332191a688c1f969d80830557be9b 2024-11-21T11:16:17,650 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20b332191a688c1f969d80830557be9b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T11:16:17,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=A 2024-11-21T11:16:17,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:17,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=B 2024-11-21T11:16:17,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:17,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=C 2024-11-21T11:16:17,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:17,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/4c2ca8394e4f40e5a023ed8cd3634ccb is 50, key is test_row_0/A:col10/1732187777648/Put/seqid=0 2024-11-21T11:16:17,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742363_1539 (size=14741) 2024-11-21T11:16:17,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:17,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187837678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:17,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:17,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:17,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187837681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:17,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:17,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:17,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187837683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:17,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:17,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187837684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:17,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:17,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:17,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:17,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:17,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187837796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:17,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187837796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:17,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:17,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187837805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:17,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:17,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187837805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:17,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:17,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:17,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:17,930 INFO [Thread-2071 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2071. 2024-11-21T11:16:17,930 DEBUG [Thread-2071 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:16:17,930 DEBUG [Thread-2071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:17,930 DEBUG [Thread-2071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:17,930 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:16:17,936 INFO [Thread-2065 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2065. 2024-11-21T11:16:17,937 DEBUG [Thread-2065 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:16:17,937 DEBUG [Thread-2065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:17,937 DEBUG [Thread-2065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:17,937 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:16:17,939 INFO [Thread-2067 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2067. 2024-11-21T11:16:17,939 DEBUG [Thread-2067 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:16:17,939 DEBUG [Thread-2067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:17,939 DEBUG [Thread-2067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:17,939 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:16:17,941 INFO [Thread-2069 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2069. 2024-11-21T11:16:17,941 DEBUG [Thread-2069 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:16:17,941 DEBUG [Thread-2069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:17,941 DEBUG [Thread-2069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:17,942 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:16:17,946 INFO [Thread-2063 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2063. 2024-11-21T11:16:17,946 DEBUG [Thread-2063 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:16:17,946 DEBUG [Thread-2063 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:17,946 DEBUG [Thread-2063 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:17,946 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:16:18,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:18,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:18,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187838016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187838016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187838016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:18,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187838016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:18,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:18,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:18,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:18,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:18,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:18,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:18,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:18,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/4c2ca8394e4f40e5a023ed8cd3634ccb 2024-11-21T11:16:18,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/362177272767469cab108756f14c55be is 50, key is test_row_0/B:col10/1732187777648/Put/seqid=0 2024-11-21T11:16:18,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742364_1540 (size=12301) 2024-11-21T11:16:18,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:18,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:18,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187838325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187838325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187838325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:18,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187838325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:18,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/362177272767469cab108756f14c55be 2024-11-21T11:16:18,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/3321190d0a1949a9b2bd1924ad140db4 is 50, key is test_row_0/C:col10/1732187777648/Put/seqid=0 2024-11-21T11:16:18,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742365_1541 (size=12301) 2024-11-21T11:16:18,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:18,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51738 deadline: 1732187838535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:18,536 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_1' in 20b332191a688c1f969d80830557be9b of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18257 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:16:18,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:18,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:18,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-21T11:16:18,606 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:16:18,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:18,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:18,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51784 deadline: 1732187838835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51740 deadline: 1732187838835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1732187838835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:18,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51722 deadline: 1732187838836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:18,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:18,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:18,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:18,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:18,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:18,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:18,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:18,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:18,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:18,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 , the old value is region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:18,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:18,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., hostname=7b462513bfc2,33417,1732187618276, seqNum=2 because the exception is null or not the one we care about 2024-11-21T11:16:18,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/3321190d0a1949a9b2bd1924ad140db4 2024-11-21T11:16:18,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/4c2ca8394e4f40e5a023ed8cd3634ccb as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/4c2ca8394e4f40e5a023ed8cd3634ccb 2024-11-21T11:16:18,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/4c2ca8394e4f40e5a023ed8cd3634ccb, entries=200, sequenceid=390, filesize=14.4 K 2024-11-21T11:16:18,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/362177272767469cab108756f14c55be as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/362177272767469cab108756f14c55be 2024-11-21T11:16:18,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/362177272767469cab108756f14c55be, entries=150, sequenceid=390, filesize=12.0 K 2024-11-21T11:16:18,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/3321190d0a1949a9b2bd1924ad140db4 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/3321190d0a1949a9b2bd1924ad140db4 2024-11-21T11:16:18,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/3321190d0a1949a9b2bd1924ad140db4, entries=150, sequenceid=390, filesize=12.0 K 2024-11-21T11:16:18,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 20b332191a688c1f969d80830557be9b in 1240ms, sequenceid=390, compaction requested=true 2024-11-21T11:16:18,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:18,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20b332191a688c1f969d80830557be9b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:16:18,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:18,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20b332191a688c1f969d80830557be9b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:16:18,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:18,889 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:16:18,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20b332191a688c1f969d80830557be9b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:16:18,889 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:16:18,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:16:18,889 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:16:18,889 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:16:18,890 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 20b332191a688c1f969d80830557be9b/A is initiating minor compaction (all files) 2024-11-21T11:16:18,890 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 20b332191a688c1f969d80830557be9b/B is initiating minor compaction (all files) 2024-11-21T11:16:18,890 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20b332191a688c1f969d80830557be9b/A in TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:18,890 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20b332191a688c1f969d80830557be9b/B in TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:18,890 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/c2ef7e9e39d04256816c0d2d7d422304, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/f0368482359540b8ac22e9ed783b33ea, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/4c2ca8394e4f40e5a023ed8cd3634ccb] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp, totalSize=39.3 K 2024-11-21T11:16:18,890 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/722bd9f86f42442bbac8ff7ce7390268, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/364b3d16ddeb4c86a81ae6e428cbcfa9, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/362177272767469cab108756f14c55be] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp, totalSize=36.9 K 2024-11-21T11:16:18,890 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 722bd9f86f42442bbac8ff7ce7390268, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1732187774421 2024-11-21T11:16:18,890 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2ef7e9e39d04256816c0d2d7d422304, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1732187774421 2024-11-21T11:16:18,890 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 364b3d16ddeb4c86a81ae6e428cbcfa9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1732187775808 2024-11-21T11:16:18,890 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0368482359540b8ac22e9ed783b33ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1732187775808 2024-11-21T11:16:18,890 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 362177272767469cab108756f14c55be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1732187776992 2024-11-21T11:16:18,890 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c2ca8394e4f40e5a023ed8cd3634ccb, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1732187776992 2024-11-21T11:16:18,896 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20b332191a688c1f969d80830557be9b#B#compaction#461 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:18,896 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20b332191a688c1f969d80830557be9b#A#compaction#462 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:18,896 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/490bbf71f2e842cd8d8bb7e15f0ba149 is 50, key is test_row_0/A:col10/1732187777648/Put/seqid=0 2024-11-21T11:16:18,896 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/94a5f5fbfb704a918fb7f10462ab1776 is 50, key is test_row_0/B:col10/1732187777648/Put/seqid=0 2024-11-21T11:16:18,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742366_1542 (size=13255) 2024-11-21T11:16:18,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742367_1543 (size=13255) 2024-11-21T11:16:18,907 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/490bbf71f2e842cd8d8bb7e15f0ba149 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/490bbf71f2e842cd8d8bb7e15f0ba149 2024-11-21T11:16:18,910 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20b332191a688c1f969d80830557be9b/A of 20b332191a688c1f969d80830557be9b into 490bbf71f2e842cd8d8bb7e15f0ba149(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:18,910 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:18,910 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., storeName=20b332191a688c1f969d80830557be9b/A, priority=13, startTime=1732187778889; duration=0sec 2024-11-21T11:16:18,911 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:16:18,911 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20b332191a688c1f969d80830557be9b:A 2024-11-21T11:16:18,911 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:16:18,911 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:16:18,911 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 20b332191a688c1f969d80830557be9b/C is initiating minor compaction (all files) 2024-11-21T11:16:18,911 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20b332191a688c1f969d80830557be9b/C in TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:18,912 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/8c90aecf6ea440a7b3f020cd99d8d4ab, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/2a492560610e4dd4945892f641cde478, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/3321190d0a1949a9b2bd1924ad140db4] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp, totalSize=36.9 K 2024-11-21T11:16:18,912 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c90aecf6ea440a7b3f020cd99d8d4ab, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1732187774421 2024-11-21T11:16:18,912 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a492560610e4dd4945892f641cde478, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1732187775808 2024-11-21T11:16:18,912 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3321190d0a1949a9b2bd1924ad140db4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1732187776992 2024-11-21T11:16:18,918 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20b332191a688c1f969d80830557be9b#C#compaction#463 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:18,918 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/6d9219ec629a48a29d185f6a1e7699ca is 50, key is test_row_0/C:col10/1732187777648/Put/seqid=0 2024-11-21T11:16:18,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742368_1544 (size=13255) 2024-11-21T11:16:19,305 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/94a5f5fbfb704a918fb7f10462ab1776 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/94a5f5fbfb704a918fb7f10462ab1776 2024-11-21T11:16:19,308 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20b332191a688c1f969d80830557be9b/B of 20b332191a688c1f969d80830557be9b into 94a5f5fbfb704a918fb7f10462ab1776(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:19,308 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:19,308 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., storeName=20b332191a688c1f969d80830557be9b/B, priority=13, startTime=1732187778889; duration=0sec 2024-11-21T11:16:19,308 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:19,308 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20b332191a688c1f969d80830557be9b:B 2024-11-21T11:16:19,325 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/6d9219ec629a48a29d185f6a1e7699ca as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/6d9219ec629a48a29d185f6a1e7699ca 2024-11-21T11:16:19,328 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20b332191a688c1f969d80830557be9b/C of 20b332191a688c1f969d80830557be9b into 6d9219ec629a48a29d185f6a1e7699ca(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:19,328 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:19,328 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., storeName=20b332191a688c1f969d80830557be9b/C, priority=13, startTime=1732187778889; duration=0sec 2024-11-21T11:16:19,328 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:19,328 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20b332191a688c1f969d80830557be9b:C 2024-11-21T11:16:19,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 20b332191a688c1f969d80830557be9b 2024-11-21T11:16:19,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20b332191a688c1f969d80830557be9b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-21T11:16:19,847 INFO [Thread-2052 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2052. 2024-11-21T11:16:19,847 INFO [Thread-2056 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2056. 2024-11-21T11:16:19,847 DEBUG [Thread-2052 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:16:19,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=A 2024-11-21T11:16:19,847 DEBUG [Thread-2056 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:16:19,847 DEBUG [Thread-2052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:19,847 DEBUG [Thread-2056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:19,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:19,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=B 2024-11-21T11:16:19,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:19,847 DEBUG [Thread-2052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:19,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=C 2024-11-21T11:16:19,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:19,847 DEBUG [Thread-2056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:19,847 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:16:19,847 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:16:19,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/63559cf3e09346428635401b79460679 is 50, key is test_row_0/A:col10/1732187779845/Put/seqid=0 2024-11-21T11:16:19,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742369_1545 (size=12301) 2024-11-21T11:16:19,857 INFO [Thread-2054 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2054. 2024-11-21T11:16:19,857 INFO [Thread-2060 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2060. 2024-11-21T11:16:19,857 DEBUG [Thread-2054 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:16:19,857 DEBUG [Thread-2060 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:16:19,857 DEBUG [Thread-2054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:19,857 DEBUG [Thread-2060 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:19,857 DEBUG [Thread-2054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:19,857 DEBUG [Thread-2060 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:19,857 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:16:19,857 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:16:20,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/63559cf3e09346428635401b79460679 2024-11-21T11:16:20,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/f571dde6319744ef8eb1e7e4d194a8fc is 50, key is test_row_0/B:col10/1732187779845/Put/seqid=0 2024-11-21T11:16:20,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742370_1546 (size=12301) 2024-11-21T11:16:20,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/f571dde6319744ef8eb1e7e4d194a8fc 2024-11-21T11:16:20,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/502ea9e2e8d74b1c991b368a51febd85 is 50, key is test_row_0/C:col10/1732187779845/Put/seqid=0 2024-11-21T11:16:20,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742371_1547 (size=12301) 2024-11-21T11:16:21,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/502ea9e2e8d74b1c991b368a51febd85 2024-11-21T11:16:21,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/63559cf3e09346428635401b79460679 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/63559cf3e09346428635401b79460679 2024-11-21T11:16:21,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/63559cf3e09346428635401b79460679, entries=150, sequenceid=413, filesize=12.0 K 2024-11-21T11:16:21,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/f571dde6319744ef8eb1e7e4d194a8fc as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/f571dde6319744ef8eb1e7e4d194a8fc 2024-11-21T11:16:21,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/f571dde6319744ef8eb1e7e4d194a8fc, entries=150, sequenceid=413, filesize=12.0 K 2024-11-21T11:16:21,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/502ea9e2e8d74b1c991b368a51febd85 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/502ea9e2e8d74b1c991b368a51febd85 2024-11-21T11:16:21,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/502ea9e2e8d74b1c991b368a51febd85, entries=150, sequenceid=413, filesize=12.0 K 2024-11-21T11:16:21,085 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=13.42 KB/13740 for 20b332191a688c1f969d80830557be9b in 1238ms, sequenceid=413, compaction requested=false 2024-11-21T11:16:21,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20b332191a688c1f969d80830557be9b: 2024-11-21T11:16:28,557 INFO [Thread-2058 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2058. 2024-11-21T11:16:28,557 DEBUG [Thread-2058 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:16:28,557 DEBUG [Thread-2058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:28,557 DEBUG [Thread-2058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:28,557 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:16:28,557 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T11:16:28,557 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-21T11:16:28,557 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-21T11:16:28,557 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-21T11:16:28,557 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-21T11:16:28,557 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-21T11:16:28,557 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T11:16:28,557 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T11:16:28,557 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1201 2024-11-21T11:16:28,557 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3602 rows 2024-11-21T11:16:28,557 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1203 2024-11-21T11:16:28,557 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3608 rows 2024-11-21T11:16:28,557 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1200 2024-11-21T11:16:28,557 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3599 rows 2024-11-21T11:16:28,557 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1203 2024-11-21T11:16:28,557 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3606 rows 2024-11-21T11:16:28,557 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1202 2024-11-21T11:16:28,557 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3600 rows 2024-11-21T11:16:28,558 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T11:16:28,558 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T11:16:28,558 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:28,558 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:28,558 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:16:28,558 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T11:16:28,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T11:16:28,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-21T11:16:28,561 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732187788561"}]},"ts":"1732187788561"} 2024-11-21T11:16:28,562 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T11:16:28,562 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T11:16:28,562 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T11:16:28,563 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=20b332191a688c1f969d80830557be9b, UNASSIGN}] 2024-11-21T11:16:28,564 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=20b332191a688c1f969d80830557be9b, UNASSIGN 2024-11-21T11:16:28,564 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=20b332191a688c1f969d80830557be9b, regionState=CLOSING, regionLocation=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:28,565 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=20b332191a688c1f969d80830557be9b, UNASSIGN because future has completed 2024-11-21T11:16:28,566 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T11:16:28,566 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276}] 2024-11-21T11:16:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-21T11:16:28,718 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close 20b332191a688c1f969d80830557be9b 2024-11-21T11:16:28,718 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T11:16:28,718 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing 20b332191a688c1f969d80830557be9b, disabling compactions & flushes 2024-11-21T11:16:28,718 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:28,718 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:28,718 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. after waiting 0 ms 2024-11-21T11:16:28,718 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:28,718 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(2902): Flushing 20b332191a688c1f969d80830557be9b 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-21T11:16:28,718 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=A 2024-11-21T11:16:28,718 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:28,718 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=B 2024-11-21T11:16:28,718 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:28,718 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20b332191a688c1f969d80830557be9b, store=C 2024-11-21T11:16:28,718 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:28,721 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/f1302a0e019d4c75ae43d15ebd5c9f2e is 50, key is test_row_1/A:col10/1732187788555/Put/seqid=0 2024-11-21T11:16:28,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742372_1548 (size=9857) 2024-11-21T11:16:28,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-21T11:16:29,125 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/f1302a0e019d4c75ae43d15ebd5c9f2e 2024-11-21T11:16:29,131 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/0a2a3a020fb24841b40ee762b17b6b46 is 50, key is test_row_1/B:col10/1732187788555/Put/seqid=0 2024-11-21T11:16:29,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742373_1549 (size=9857) 2024-11-21T11:16:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-21T11:16:29,534 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/0a2a3a020fb24841b40ee762b17b6b46 2024-11-21T11:16:29,540 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/df721a5aceb0412fbdf757bddbb9dd1b is 50, key is test_row_1/C:col10/1732187788555/Put/seqid=0 2024-11-21T11:16:29,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742374_1550 (size=9857) 2024-11-21T11:16:29,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-21T11:16:29,943 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/df721a5aceb0412fbdf757bddbb9dd1b 2024-11-21T11:16:29,946 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/A/f1302a0e019d4c75ae43d15ebd5c9f2e as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/f1302a0e019d4c75ae43d15ebd5c9f2e 2024-11-21T11:16:29,949 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/f1302a0e019d4c75ae43d15ebd5c9f2e, entries=100, sequenceid=419, filesize=9.6 K 2024-11-21T11:16:29,949 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/B/0a2a3a020fb24841b40ee762b17b6b46 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/0a2a3a020fb24841b40ee762b17b6b46 2024-11-21T11:16:29,952 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/0a2a3a020fb24841b40ee762b17b6b46, entries=100, sequenceid=419, filesize=9.6 K 2024-11-21T11:16:29,953 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/.tmp/C/df721a5aceb0412fbdf757bddbb9dd1b as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/df721a5aceb0412fbdf757bddbb9dd1b 2024-11-21T11:16:29,955 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/df721a5aceb0412fbdf757bddbb9dd1b, entries=100, sequenceid=419, filesize=9.6 K 2024-11-21T11:16:29,956 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 20b332191a688c1f969d80830557be9b in 1238ms, sequenceid=419, compaction requested=true 2024-11-21T11:16:29,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/fbdff1fd956f44fdac43ab95213b8a8d, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/3767dba113e34163954c9dbee575a1d5, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/dd04a7b4837249babe50d448c4f6aede, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/cc79ab9238de4b9cb5defaa6554fca3d, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/5b44e85c8a4d40acbccb62799f6c3def, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/a3d5a463df1943cdb8f307706bffe136, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/e5b0746a695044b99babda4fea57b588, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/830e4a5db83f43d8b4c96f64e04e83cd, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/5b778ca750704fdea3f04969692592c1, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/578a195ab9614f50b44ca1150f658f60, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/3fe50ccdda6b4dbeb4c13b1883dea816, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/9c9e8258c950494b916f0789c4e1355a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/cd3bab54c63445dfa8be79c7578c26aa, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/b1c642677b17427f96a946fba4791682, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/777d4b4e6cb04cf9b0e38518bac0c39e, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/2102570a80734bdbac2d32c4157dc60b, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/aba528cf33554b31a9a5589debe0af0a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/e3ec4ad6a93d4e2fb9668aee151c8f06, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/2892e91db57b4ded99cffa634cb767f9, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/48e6852edaef446f85c95c0082f68dc7, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/12c6bfe70fc44db9b1ed8f801b67fe72, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/4579fe9d3ea74633ae07f99e0f6cec42, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/792acd62b35b472dbcd2d6b9cafa95e8, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/8703b44b876448e892f5a0b20249c3d7, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/c2ef7e9e39d04256816c0d2d7d422304, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/edb9262a57364ee2b940919baa6864ec, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/f0368482359540b8ac22e9ed783b33ea, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/4c2ca8394e4f40e5a023ed8cd3634ccb] to archive 2024-11-21T11:16:29,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T11:16:29,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/fbdff1fd956f44fdac43ab95213b8a8d to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/fbdff1fd956f44fdac43ab95213b8a8d 2024-11-21T11:16:29,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/3767dba113e34163954c9dbee575a1d5 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/3767dba113e34163954c9dbee575a1d5 2024-11-21T11:16:29,960 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/dd04a7b4837249babe50d448c4f6aede to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/dd04a7b4837249babe50d448c4f6aede 2024-11-21T11:16:29,961 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/cc79ab9238de4b9cb5defaa6554fca3d to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/cc79ab9238de4b9cb5defaa6554fca3d 2024-11-21T11:16:29,961 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/5b44e85c8a4d40acbccb62799f6c3def to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/5b44e85c8a4d40acbccb62799f6c3def 2024-11-21T11:16:29,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/a3d5a463df1943cdb8f307706bffe136 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/a3d5a463df1943cdb8f307706bffe136 2024-11-21T11:16:29,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/e5b0746a695044b99babda4fea57b588 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/e5b0746a695044b99babda4fea57b588 2024-11-21T11:16:29,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/830e4a5db83f43d8b4c96f64e04e83cd to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/830e4a5db83f43d8b4c96f64e04e83cd 2024-11-21T11:16:29,964 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/5b778ca750704fdea3f04969692592c1 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/5b778ca750704fdea3f04969692592c1 2024-11-21T11:16:29,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/578a195ab9614f50b44ca1150f658f60 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/578a195ab9614f50b44ca1150f658f60 2024-11-21T11:16:29,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/3fe50ccdda6b4dbeb4c13b1883dea816 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/3fe50ccdda6b4dbeb4c13b1883dea816 2024-11-21T11:16:29,966 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/9c9e8258c950494b916f0789c4e1355a to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/9c9e8258c950494b916f0789c4e1355a 2024-11-21T11:16:29,967 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/cd3bab54c63445dfa8be79c7578c26aa to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/cd3bab54c63445dfa8be79c7578c26aa 2024-11-21T11:16:29,967 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/b1c642677b17427f96a946fba4791682 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/b1c642677b17427f96a946fba4791682 2024-11-21T11:16:29,968 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/777d4b4e6cb04cf9b0e38518bac0c39e to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/777d4b4e6cb04cf9b0e38518bac0c39e 2024-11-21T11:16:29,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/2102570a80734bdbac2d32c4157dc60b to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/2102570a80734bdbac2d32c4157dc60b 2024-11-21T11:16:29,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/aba528cf33554b31a9a5589debe0af0a to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/aba528cf33554b31a9a5589debe0af0a 2024-11-21T11:16:29,970 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/e3ec4ad6a93d4e2fb9668aee151c8f06 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/e3ec4ad6a93d4e2fb9668aee151c8f06 2024-11-21T11:16:29,971 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/2892e91db57b4ded99cffa634cb767f9 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/2892e91db57b4ded99cffa634cb767f9 2024-11-21T11:16:29,972 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/48e6852edaef446f85c95c0082f68dc7 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/48e6852edaef446f85c95c0082f68dc7 2024-11-21T11:16:29,973 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/12c6bfe70fc44db9b1ed8f801b67fe72 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/12c6bfe70fc44db9b1ed8f801b67fe72 2024-11-21T11:16:29,973 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/4579fe9d3ea74633ae07f99e0f6cec42 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/4579fe9d3ea74633ae07f99e0f6cec42 2024-11-21T11:16:29,974 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/792acd62b35b472dbcd2d6b9cafa95e8 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/792acd62b35b472dbcd2d6b9cafa95e8 2024-11-21T11:16:29,975 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/8703b44b876448e892f5a0b20249c3d7 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/8703b44b876448e892f5a0b20249c3d7 2024-11-21T11:16:29,976 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/c2ef7e9e39d04256816c0d2d7d422304 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/c2ef7e9e39d04256816c0d2d7d422304 2024-11-21T11:16:29,977 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/edb9262a57364ee2b940919baa6864ec to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/edb9262a57364ee2b940919baa6864ec 2024-11-21T11:16:29,978 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/f0368482359540b8ac22e9ed783b33ea to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/f0368482359540b8ac22e9ed783b33ea 2024-11-21T11:16:29,979 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/4c2ca8394e4f40e5a023ed8cd3634ccb to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/4c2ca8394e4f40e5a023ed8cd3634ccb 2024-11-21T11:16:29,980 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/0a3d15e49cea4e33b65b279ecf5abc72, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/ebdbee38f0cf48e190c8bf1b9b6c01e3, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/57cc2fa179c34c17a748ecc2553b06d6, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/6d293e0286934c068393a95ed6c6a867, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/00256a54c8af46bbaf0952a74321a657, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/f062b96f97194c50a2b5fd2642e4e1c5, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/b76c10047a8142678e639a92143cf523, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/066c544e311147e6939659de48f90b83, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/e316d953f0974427b509dd2821cdee6e, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/fdd5e8bdafeb43e4a641d34629fc0f17, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/6282209cf24b4e9c914b592093841257, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/55d6d212d46b489d909013db980dbcd2, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/5b4bdca400a84c8aaece569d4622bd8f, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/8772c60a91024409a9ece10ad6471d31, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/cf5dc5a62f014d49903d20ed9a90f3aa, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/95db0830809c4329b7b9c8aac332ecc4, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/5e98377b7ab24a9bb400525a5af3f347, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/2aa3d0c257d34fd1b2763c53f52bde9b, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/4ddf85f57d6243658d05faf91ce0284a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/c0d47974745d45baa7f510327d2a61ef, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/b00caab6b24b49248b5fcf32a72e10de, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/09c3c617c04a4abbb6dc222319a62b74, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/ff4882fb22064b158f466fef1b63fa88, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/bdf18b8d175948b4b3ea6c8e3879190a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/722bd9f86f42442bbac8ff7ce7390268, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/94574e1a1c50449498ce268e1c032f46, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/364b3d16ddeb4c86a81ae6e428cbcfa9, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/362177272767469cab108756f14c55be] to archive 2024-11-21T11:16:29,981 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T11:16:29,982 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/0a3d15e49cea4e33b65b279ecf5abc72 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/0a3d15e49cea4e33b65b279ecf5abc72 2024-11-21T11:16:29,982 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/ebdbee38f0cf48e190c8bf1b9b6c01e3 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/ebdbee38f0cf48e190c8bf1b9b6c01e3 2024-11-21T11:16:29,983 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/57cc2fa179c34c17a748ecc2553b06d6 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/57cc2fa179c34c17a748ecc2553b06d6 2024-11-21T11:16:29,984 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/6d293e0286934c068393a95ed6c6a867 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/6d293e0286934c068393a95ed6c6a867 2024-11-21T11:16:29,985 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/00256a54c8af46bbaf0952a74321a657 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/00256a54c8af46bbaf0952a74321a657 2024-11-21T11:16:29,986 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/f062b96f97194c50a2b5fd2642e4e1c5 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/f062b96f97194c50a2b5fd2642e4e1c5 2024-11-21T11:16:29,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/b76c10047a8142678e639a92143cf523 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/b76c10047a8142678e639a92143cf523 2024-11-21T11:16:29,988 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/066c544e311147e6939659de48f90b83 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/066c544e311147e6939659de48f90b83 2024-11-21T11:16:29,988 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/e316d953f0974427b509dd2821cdee6e to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/e316d953f0974427b509dd2821cdee6e 2024-11-21T11:16:29,989 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/fdd5e8bdafeb43e4a641d34629fc0f17 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/fdd5e8bdafeb43e4a641d34629fc0f17 2024-11-21T11:16:29,990 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/6282209cf24b4e9c914b592093841257 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/6282209cf24b4e9c914b592093841257 2024-11-21T11:16:29,991 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/55d6d212d46b489d909013db980dbcd2 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/55d6d212d46b489d909013db980dbcd2 2024-11-21T11:16:29,992 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/5b4bdca400a84c8aaece569d4622bd8f to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/5b4bdca400a84c8aaece569d4622bd8f 2024-11-21T11:16:29,993 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/8772c60a91024409a9ece10ad6471d31 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/8772c60a91024409a9ece10ad6471d31 2024-11-21T11:16:29,993 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/cf5dc5a62f014d49903d20ed9a90f3aa to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/cf5dc5a62f014d49903d20ed9a90f3aa 2024-11-21T11:16:29,994 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/95db0830809c4329b7b9c8aac332ecc4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/95db0830809c4329b7b9c8aac332ecc4 2024-11-21T11:16:29,995 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/5e98377b7ab24a9bb400525a5af3f347 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/5e98377b7ab24a9bb400525a5af3f347 2024-11-21T11:16:29,996 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/2aa3d0c257d34fd1b2763c53f52bde9b to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/2aa3d0c257d34fd1b2763c53f52bde9b 2024-11-21T11:16:29,997 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/4ddf85f57d6243658d05faf91ce0284a to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/4ddf85f57d6243658d05faf91ce0284a 2024-11-21T11:16:29,998 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/c0d47974745d45baa7f510327d2a61ef to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/c0d47974745d45baa7f510327d2a61ef 2024-11-21T11:16:29,998 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/b00caab6b24b49248b5fcf32a72e10de to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/b00caab6b24b49248b5fcf32a72e10de 2024-11-21T11:16:29,999 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/09c3c617c04a4abbb6dc222319a62b74 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/09c3c617c04a4abbb6dc222319a62b74 2024-11-21T11:16:30,000 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/ff4882fb22064b158f466fef1b63fa88 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/ff4882fb22064b158f466fef1b63fa88 2024-11-21T11:16:30,001 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/bdf18b8d175948b4b3ea6c8e3879190a to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/bdf18b8d175948b4b3ea6c8e3879190a 2024-11-21T11:16:30,001 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/722bd9f86f42442bbac8ff7ce7390268 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/722bd9f86f42442bbac8ff7ce7390268 2024-11-21T11:16:30,002 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/94574e1a1c50449498ce268e1c032f46 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/94574e1a1c50449498ce268e1c032f46 2024-11-21T11:16:30,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/364b3d16ddeb4c86a81ae6e428cbcfa9 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/364b3d16ddeb4c86a81ae6e428cbcfa9 2024-11-21T11:16:30,004 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/362177272767469cab108756f14c55be to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/362177272767469cab108756f14c55be 2024-11-21T11:16:30,005 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/fb1a187789394b4dbaf5985beda06ddd, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/2593bb338e6845b88135fc9adc838f9f, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/bce07a42a8994973bb33d2025761e6af, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/ccdfbc30d32f4af6a69655dc80c46c15, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/ff9541d8a3814db7ab2808ba00cb5b51, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/40948d7709a1423abada52037752ddb3, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/23fe4ed45fc0465b8a4ebef2f3e2d6c8, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/b697fc067ef440588804e4017605727f, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/85a389aaa2e7479fb8520a0a4c6bd675, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/536829c7a7f546ada70b42e40274726c, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/ceef9f7cc69c4653a62fe1a6a9e3d97d, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/801589f6a695477ab917884677363dce, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/a028574cd9a145c783a3e28447f6ed28, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/5d61fdcd11064d02a858a79487d5bf41, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/428a44e66eb747fb844f9c2559ad5313, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/2672f9a0ad3c42019b3bac0d496a7d26, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/d2ea107419c84d60a0c553d03df9897b, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/0cd67a696d8f42f29773bc0837dd6096, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/51d981dd86d7479f9a68cef3d703afe7, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/d1b136a85c61454ea8edb92770b03cdd, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/33b7f879b2aa464bb26e2774a45b8447, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/cd19d323871d465e8cb998bff320082c, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/3c2afef689794769bc8bc148f223cc9d, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/ab892f0ef9e4401c9873751e6bdfd900, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/8c90aecf6ea440a7b3f020cd99d8d4ab, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/b681d23f1e36423cbbd5df2d07dee372, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/2a492560610e4dd4945892f641cde478, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/3321190d0a1949a9b2bd1924ad140db4] to archive 2024-11-21T11:16:30,006 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T11:16:30,007 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/fb1a187789394b4dbaf5985beda06ddd to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/fb1a187789394b4dbaf5985beda06ddd 2024-11-21T11:16:30,008 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/2593bb338e6845b88135fc9adc838f9f to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/2593bb338e6845b88135fc9adc838f9f 2024-11-21T11:16:30,009 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/bce07a42a8994973bb33d2025761e6af to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/bce07a42a8994973bb33d2025761e6af 2024-11-21T11:16:30,009 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/ccdfbc30d32f4af6a69655dc80c46c15 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/ccdfbc30d32f4af6a69655dc80c46c15 2024-11-21T11:16:30,010 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/ff9541d8a3814db7ab2808ba00cb5b51 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/ff9541d8a3814db7ab2808ba00cb5b51 2024-11-21T11:16:30,011 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/40948d7709a1423abada52037752ddb3 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/40948d7709a1423abada52037752ddb3 2024-11-21T11:16:30,012 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/23fe4ed45fc0465b8a4ebef2f3e2d6c8 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/23fe4ed45fc0465b8a4ebef2f3e2d6c8 2024-11-21T11:16:30,013 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/b697fc067ef440588804e4017605727f to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/b697fc067ef440588804e4017605727f 2024-11-21T11:16:30,014 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/85a389aaa2e7479fb8520a0a4c6bd675 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/85a389aaa2e7479fb8520a0a4c6bd675 2024-11-21T11:16:30,014 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/536829c7a7f546ada70b42e40274726c to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/536829c7a7f546ada70b42e40274726c 2024-11-21T11:16:30,015 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/ceef9f7cc69c4653a62fe1a6a9e3d97d to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/ceef9f7cc69c4653a62fe1a6a9e3d97d 2024-11-21T11:16:30,016 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/801589f6a695477ab917884677363dce to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/801589f6a695477ab917884677363dce 2024-11-21T11:16:30,016 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/a028574cd9a145c783a3e28447f6ed28 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/a028574cd9a145c783a3e28447f6ed28 2024-11-21T11:16:30,017 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/5d61fdcd11064d02a858a79487d5bf41 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/5d61fdcd11064d02a858a79487d5bf41 2024-11-21T11:16:30,018 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/428a44e66eb747fb844f9c2559ad5313 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/428a44e66eb747fb844f9c2559ad5313 2024-11-21T11:16:30,019 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/2672f9a0ad3c42019b3bac0d496a7d26 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/2672f9a0ad3c42019b3bac0d496a7d26 2024-11-21T11:16:30,019 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/d2ea107419c84d60a0c553d03df9897b to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/d2ea107419c84d60a0c553d03df9897b 2024-11-21T11:16:30,020 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/0cd67a696d8f42f29773bc0837dd6096 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/0cd67a696d8f42f29773bc0837dd6096 2024-11-21T11:16:30,021 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/51d981dd86d7479f9a68cef3d703afe7 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/51d981dd86d7479f9a68cef3d703afe7 2024-11-21T11:16:30,022 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/d1b136a85c61454ea8edb92770b03cdd to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/d1b136a85c61454ea8edb92770b03cdd 2024-11-21T11:16:30,022 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/33b7f879b2aa464bb26e2774a45b8447 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/33b7f879b2aa464bb26e2774a45b8447 2024-11-21T11:16:30,023 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/cd19d323871d465e8cb998bff320082c to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/cd19d323871d465e8cb998bff320082c 2024-11-21T11:16:30,024 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/3c2afef689794769bc8bc148f223cc9d to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/3c2afef689794769bc8bc148f223cc9d 2024-11-21T11:16:30,024 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/ab892f0ef9e4401c9873751e6bdfd900 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/ab892f0ef9e4401c9873751e6bdfd900 2024-11-21T11:16:30,025 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/8c90aecf6ea440a7b3f020cd99d8d4ab to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/8c90aecf6ea440a7b3f020cd99d8d4ab 2024-11-21T11:16:30,026 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/b681d23f1e36423cbbd5df2d07dee372 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/b681d23f1e36423cbbd5df2d07dee372 2024-11-21T11:16:30,026 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/2a492560610e4dd4945892f641cde478 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/2a492560610e4dd4945892f641cde478 2024-11-21T11:16:30,027 DEBUG [StoreCloser-TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/3321190d0a1949a9b2bd1924ad140db4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/3321190d0a1949a9b2bd1924ad140db4 2024-11-21T11:16:30,030 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/recovered.edits/422.seqid, newMaxSeqId=422, maxSeqId=1 2024-11-21T11:16:30,031 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. 2024-11-21T11:16:30,031 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for 20b332191a688c1f969d80830557be9b: Waiting for close lock at 1732187788718Running coprocessor pre-close hooks at 1732187788718Disabling compacts and flushes for region at 1732187788718Disabling writes for close at 1732187788718Obtaining lock to block concurrent updates at 1732187788718Preparing flush snapshotting stores in 20b332191a688c1f969d80830557be9b at 1732187788718Finished memstore snapshotting TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732187788718Flushing stores of TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b. at 1732187788719 (+1 ms)Flushing 20b332191a688c1f969d80830557be9b/A: creating writer at 1732187788719Flushing 20b332191a688c1f969d80830557be9b/A: appending metadata at 1732187788721 (+2 ms)Flushing 20b332191a688c1f969d80830557be9b/A: closing flushed file at 1732187788721Flushing 20b332191a688c1f969d80830557be9b/B: creating writer at 1732187789128 (+407 ms)Flushing 20b332191a688c1f969d80830557be9b/B: appending metadata at 1732187789130 (+2 ms)Flushing 20b332191a688c1f969d80830557be9b/B: closing flushed file at 1732187789130Flushing 20b332191a688c1f969d80830557be9b/C: creating writer at 1732187789537 (+407 ms)Flushing 20b332191a688c1f969d80830557be9b/C: appending metadata at 1732187789539 (+2 ms)Flushing 20b332191a688c1f969d80830557be9b/C: closing flushed file at 1732187789539Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5df6c3fc: reopening flushed file at 1732187789946 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@326f1a92: reopening flushed file at 1732187789949 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ac15634: reopening flushed file at 1732187789952 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 20b332191a688c1f969d80830557be9b in 1238ms, sequenceid=419, compaction requested=true at 1732187789956 (+4 ms)Writing region close event to WAL at 1732187790028 (+72 ms)Running coprocessor post-close hooks at 1732187790031 (+3 ms)Closed at 1732187790031 2024-11-21T11:16:30,033 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed 20b332191a688c1f969d80830557be9b 2024-11-21T11:16:30,033 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=20b332191a688c1f969d80830557be9b, regionState=CLOSED 2024-11-21T11:16:30,034 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 because future has completed 2024-11-21T11:16:30,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-21T11:16:30,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure 20b332191a688c1f969d80830557be9b, server=7b462513bfc2,33417,1732187618276 in 1.4690 sec 2024-11-21T11:16:30,037 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-21T11:16:30,037 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=20b332191a688c1f969d80830557be9b, UNASSIGN in 1.4730 sec 2024-11-21T11:16:30,039 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-21T11:16:30,039 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4760 sec 2024-11-21T11:16:30,040 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732187790040"}]},"ts":"1732187790040"} 2024-11-21T11:16:30,041 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T11:16:30,041 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T11:16:30,043 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4840 sec 2024-11-21T11:16:30,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-21T11:16:30,706 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:16:30,706 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T11:16:30,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T11:16:30,708 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T11:16:30,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-21T11:16:30,708 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=145, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T11:16:30,710 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b 2024-11-21T11:16:30,712 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A, FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B, FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C, FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/recovered.edits] 2024-11-21T11:16:30,714 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/490bbf71f2e842cd8d8bb7e15f0ba149 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/490bbf71f2e842cd8d8bb7e15f0ba149 2024-11-21T11:16:30,714 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/63559cf3e09346428635401b79460679 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/63559cf3e09346428635401b79460679 2024-11-21T11:16:30,715 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/f1302a0e019d4c75ae43d15ebd5c9f2e to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/A/f1302a0e019d4c75ae43d15ebd5c9f2e 2024-11-21T11:16:30,717 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/0a2a3a020fb24841b40ee762b17b6b46 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/0a2a3a020fb24841b40ee762b17b6b46 2024-11-21T11:16:30,718 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/94a5f5fbfb704a918fb7f10462ab1776 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/94a5f5fbfb704a918fb7f10462ab1776 2024-11-21T11:16:30,719 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/f571dde6319744ef8eb1e7e4d194a8fc to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/B/f571dde6319744ef8eb1e7e4d194a8fc 2024-11-21T11:16:30,721 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/502ea9e2e8d74b1c991b368a51febd85 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/502ea9e2e8d74b1c991b368a51febd85 2024-11-21T11:16:30,722 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/6d9219ec629a48a29d185f6a1e7699ca to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/6d9219ec629a48a29d185f6a1e7699ca 2024-11-21T11:16:30,722 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/df721a5aceb0412fbdf757bddbb9dd1b to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/C/df721a5aceb0412fbdf757bddbb9dd1b 2024-11-21T11:16:30,724 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/recovered.edits/422.seqid to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b/recovered.edits/422.seqid 2024-11-21T11:16:30,725 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/20b332191a688c1f969d80830557be9b 2024-11-21T11:16:30,725 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T11:16:30,726 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=145, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T11:16:30,728 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T11:16:30,729 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T11:16:30,730 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=145, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T11:16:30,730 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T11:16:30,730 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732187790730"}]},"ts":"9223372036854775807"} 2024-11-21T11:16:30,732 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T11:16:30,732 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 20b332191a688c1f969d80830557be9b, NAME => 'TestAcidGuarantees,,1732187755735.20b332191a688c1f969d80830557be9b.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T11:16:30,732 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T11:16:30,732 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732187790732"}]},"ts":"9223372036854775807"} 2024-11-21T11:16:30,733 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T11:16:30,734 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=145, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T11:16:30,734 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 28 msec 2024-11-21T11:16:30,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-21T11:16:30,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T11:16:30,815 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:16:30,825 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=246 (was 249), OpenFileDescriptor=441 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=315 (was 368), ProcessCount=11 (was 11), AvailableMemoryMB=4512 (was 4600) 2024-11-21T11:16:30,833 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=246, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=315, ProcessCount=11, AvailableMemoryMB=4511 2024-11-21T11:16:30,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T11:16:30,834 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T11:16:30,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T11:16:30,836 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T11:16:30,836 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:30,836 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 146 2024-11-21T11:16:30,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-21T11:16:30,836 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T11:16:30,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742375_1551 (size=1061) 2024-11-21T11:16:30,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-21T11:16:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-21T11:16:31,243 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322 2024-11-21T11:16:31,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742376_1552 (size=53) 2024-11-21T11:16:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-21T11:16:31,648 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:16:31,648 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 0e38bed8d085aac2e0e9aec5f988def4, disabling compactions & flushes 2024-11-21T11:16:31,648 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:31,648 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:31,648 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. after waiting 0 ms 2024-11-21T11:16:31,648 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:31,648 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:31,648 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0e38bed8d085aac2e0e9aec5f988def4: Waiting for close lock at 1732187791648Disabling compacts and flushes for region at 1732187791648Disabling writes for close at 1732187791648Writing region close event to WAL at 1732187791648Closed at 1732187791648 2024-11-21T11:16:31,649 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T11:16:31,650 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732187791649"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732187791649"}]},"ts":"1732187791649"} 2024-11-21T11:16:31,651 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T11:16:31,651 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T11:16:31,651 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732187791651"}]},"ts":"1732187791651"} 2024-11-21T11:16:31,652 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T11:16:31,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e38bed8d085aac2e0e9aec5f988def4, ASSIGN}] 2024-11-21T11:16:31,653 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e38bed8d085aac2e0e9aec5f988def4, ASSIGN 2024-11-21T11:16:31,654 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e38bed8d085aac2e0e9aec5f988def4, ASSIGN; state=OFFLINE, location=7b462513bfc2,33417,1732187618276; forceNewPlan=false, retain=false 2024-11-21T11:16:31,804 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=0e38bed8d085aac2e0e9aec5f988def4, regionState=OPENING, regionLocation=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:31,806 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e38bed8d085aac2e0e9aec5f988def4, ASSIGN because future has completed 2024-11-21T11:16:31,807 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276}] 2024-11-21T11:16:31,961 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:31,961 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7752): Opening region: {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} 2024-11-21T11:16:31,961 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:31,961 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:16:31,961 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7794): checking encryption for 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:31,961 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7797): checking classloading for 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:31,962 INFO [StoreOpener-0e38bed8d085aac2e0e9aec5f988def4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:31,963 INFO [StoreOpener-0e38bed8d085aac2e0e9aec5f988def4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T11:16:31,964 INFO [StoreOpener-0e38bed8d085aac2e0e9aec5f988def4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0e38bed8d085aac2e0e9aec5f988def4 columnFamilyName A 2024-11-21T11:16:31,964 DEBUG [StoreOpener-0e38bed8d085aac2e0e9aec5f988def4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:31,964 INFO [StoreOpener-0e38bed8d085aac2e0e9aec5f988def4-1 {}] regionserver.HStore(327): Store=0e38bed8d085aac2e0e9aec5f988def4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:16:31,964 INFO [StoreOpener-0e38bed8d085aac2e0e9aec5f988def4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:31,965 INFO [StoreOpener-0e38bed8d085aac2e0e9aec5f988def4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T11:16:31,965 INFO [StoreOpener-0e38bed8d085aac2e0e9aec5f988def4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0e38bed8d085aac2e0e9aec5f988def4 columnFamilyName B 2024-11-21T11:16:31,965 DEBUG [StoreOpener-0e38bed8d085aac2e0e9aec5f988def4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:31,965 INFO [StoreOpener-0e38bed8d085aac2e0e9aec5f988def4-1 {}] regionserver.HStore(327): Store=0e38bed8d085aac2e0e9aec5f988def4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:16:31,965 INFO [StoreOpener-0e38bed8d085aac2e0e9aec5f988def4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:31,966 INFO [StoreOpener-0e38bed8d085aac2e0e9aec5f988def4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T11:16:31,966 INFO [StoreOpener-0e38bed8d085aac2e0e9aec5f988def4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0e38bed8d085aac2e0e9aec5f988def4 columnFamilyName C 2024-11-21T11:16:31,966 DEBUG [StoreOpener-0e38bed8d085aac2e0e9aec5f988def4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:31,966 INFO [StoreOpener-0e38bed8d085aac2e0e9aec5f988def4-1 {}] regionserver.HStore(327): Store=0e38bed8d085aac2e0e9aec5f988def4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:16:31,967 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:31,967 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1038): replaying wal for 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:31,967 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:31,967 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:31,968 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1048): stopping wal replay for 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:31,968 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1060): Cleaning up temporary data for 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:31,968 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T11:16:31,970 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1093): writing seq id for 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:31,971 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T11:16:31,971 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1114): Opened 0e38bed8d085aac2e0e9aec5f988def4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73878921, jitterRate=0.10088171064853668}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T11:16:31,971 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:31,972 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1006): Region open journal for 0e38bed8d085aac2e0e9aec5f988def4: Running coprocessor pre-open hook at 1732187791961Writing region info on filesystem at 1732187791961Initializing all the Stores at 1732187791962 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187791962Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187791962Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187791962Cleaning up temporary data from old regions at 1732187791968 (+6 ms)Running coprocessor post-open hooks at 1732187791971 (+3 ms)Region opened successfully at 1732187791972 (+1 ms) 2024-11-21T11:16:31,972 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., pid=148, masterSystemTime=1732187791958 2024-11-21T11:16:31,974 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:31,974 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:31,974 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=0e38bed8d085aac2e0e9aec5f988def4, regionState=OPEN, openSeqNum=2, regionLocation=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:31,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 because future has completed 2024-11-21T11:16:31,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-21T11:16:31,978 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-21T11:16:31,978 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure 0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 in 170 msec 2024-11-21T11:16:31,979 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-21T11:16:31,979 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e38bed8d085aac2e0e9aec5f988def4, ASSIGN in 325 msec 2024-11-21T11:16:31,979 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T11:16:31,980 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732187791980"}]},"ts":"1732187791980"} 2024-11-21T11:16:31,981 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T11:16:31,981 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T11:16:31,982 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1470 sec 2024-11-21T11:16:32,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-21T11:16:32,986 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:16:32,987 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@145ad34b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:16:32,987 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:16:32,987 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:16:32,988 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:16:32,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:16:32,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:16:32,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1db4f580, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:16:32,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:16:32,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:16:32,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:32,990 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:16:32,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24132fb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:16:32,990 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:16:32,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:16:32,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:16:32,992 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:16:32,993 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T11:16:32,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 7b462513bfc2,36543,1732187617563 2024-11-21T11:16:32,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@72f49f34 2024-11-21T11:16:32,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T11:16:32,994 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T11:16:32,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T11:16:32,995 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T11:16:32,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-21T11:16:33,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742377_1553 (size=1097) 2024-11-21T11:16:33,406 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-21T11:16:33,406 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-21T11:16:33,408 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T11:16:33,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e38bed8d085aac2e0e9aec5f988def4, REOPEN/MOVE}] 2024-11-21T11:16:33,410 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e38bed8d085aac2e0e9aec5f988def4, REOPEN/MOVE 2024-11-21T11:16:33,411 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=0e38bed8d085aac2e0e9aec5f988def4, regionState=CLOSING, regionLocation=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:33,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e38bed8d085aac2e0e9aec5f988def4, REOPEN/MOVE because future has completed 2024-11-21T11:16:33,412 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T11:16:33,412 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276}] 2024-11-21T11:16:33,564 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(122): Close 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:33,564 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T11:16:33,564 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1722): Closing 0e38bed8d085aac2e0e9aec5f988def4, disabling compactions & flushes 2024-11-21T11:16:33,564 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:33,564 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:33,564 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. after waiting 0 ms 2024-11-21T11:16:33,564 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:33,567 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T11:16:33,567 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:33,568 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1676): Region close journal for 0e38bed8d085aac2e0e9aec5f988def4: Waiting for close lock at 1732187793564Running coprocessor pre-close hooks at 1732187793564Disabling compacts and flushes for region at 1732187793564Disabling writes for close at 1732187793564Writing region close event to WAL at 1732187793565 (+1 ms)Running coprocessor post-close hooks at 1732187793567 (+2 ms)Closed at 1732187793567 2024-11-21T11:16:33,568 WARN [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegionServer(3299): Not adding moved region record: 0e38bed8d085aac2e0e9aec5f988def4 to self. 2024-11-21T11:16:33,569 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(157): Closed 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:33,569 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=0e38bed8d085aac2e0e9aec5f988def4, regionState=CLOSED 2024-11-21T11:16:33,570 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 because future has completed 2024-11-21T11:16:33,572 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-21T11:16:33,572 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; CloseRegionProcedure 0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 in 159 msec 2024-11-21T11:16:33,572 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e38bed8d085aac2e0e9aec5f988def4, REOPEN/MOVE; state=CLOSED, location=7b462513bfc2,33417,1732187618276; forceNewPlan=false, retain=true 2024-11-21T11:16:33,723 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=0e38bed8d085aac2e0e9aec5f988def4, regionState=OPENING, regionLocation=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:33,724 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e38bed8d085aac2e0e9aec5f988def4, REOPEN/MOVE because future has completed 2024-11-21T11:16:33,725 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276}] 2024-11-21T11:16:33,879 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:33,879 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7752): Opening region: {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} 2024-11-21T11:16:33,879 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:33,879 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:16:33,879 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7794): checking encryption for 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:33,880 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7797): checking classloading for 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:33,881 INFO [StoreOpener-0e38bed8d085aac2e0e9aec5f988def4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:33,881 INFO [StoreOpener-0e38bed8d085aac2e0e9aec5f988def4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T11:16:33,881 INFO [StoreOpener-0e38bed8d085aac2e0e9aec5f988def4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0e38bed8d085aac2e0e9aec5f988def4 columnFamilyName A 2024-11-21T11:16:33,882 DEBUG [StoreOpener-0e38bed8d085aac2e0e9aec5f988def4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:33,882 INFO [StoreOpener-0e38bed8d085aac2e0e9aec5f988def4-1 {}] regionserver.HStore(327): Store=0e38bed8d085aac2e0e9aec5f988def4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:16:33,883 INFO [StoreOpener-0e38bed8d085aac2e0e9aec5f988def4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:33,883 INFO [StoreOpener-0e38bed8d085aac2e0e9aec5f988def4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T11:16:33,883 INFO [StoreOpener-0e38bed8d085aac2e0e9aec5f988def4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0e38bed8d085aac2e0e9aec5f988def4 columnFamilyName B 2024-11-21T11:16:33,883 DEBUG [StoreOpener-0e38bed8d085aac2e0e9aec5f988def4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:33,884 INFO [StoreOpener-0e38bed8d085aac2e0e9aec5f988def4-1 {}] regionserver.HStore(327): Store=0e38bed8d085aac2e0e9aec5f988def4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:16:33,884 INFO [StoreOpener-0e38bed8d085aac2e0e9aec5f988def4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:33,884 INFO [StoreOpener-0e38bed8d085aac2e0e9aec5f988def4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T11:16:33,884 INFO [StoreOpener-0e38bed8d085aac2e0e9aec5f988def4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0e38bed8d085aac2e0e9aec5f988def4 columnFamilyName C 2024-11-21T11:16:33,884 DEBUG [StoreOpener-0e38bed8d085aac2e0e9aec5f988def4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:33,884 INFO [StoreOpener-0e38bed8d085aac2e0e9aec5f988def4-1 {}] regionserver.HStore(327): Store=0e38bed8d085aac2e0e9aec5f988def4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:16:33,885 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:33,885 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1038): replaying wal for 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:33,885 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:33,886 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:33,886 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1048): stopping wal replay for 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:33,886 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1060): Cleaning up temporary data for 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:33,887 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T11:16:33,888 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1093): writing seq id for 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:33,888 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1114): Opened 0e38bed8d085aac2e0e9aec5f988def4; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60355671, jitterRate=-0.10063041746616364}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T11:16:33,888 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:33,889 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1006): Region open journal for 0e38bed8d085aac2e0e9aec5f988def4: Running coprocessor pre-open hook at 1732187793880Writing region info on filesystem at 1732187793880Initializing all the Stores at 1732187793880Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187793880Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187793880Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187793880Cleaning up temporary data from old regions at 1732187793886 (+6 ms)Running coprocessor post-open hooks at 1732187793888 (+2 ms)Region opened successfully at 1732187793889 (+1 ms) 2024-11-21T11:16:33,889 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., pid=153, masterSystemTime=1732187793876 2024-11-21T11:16:33,891 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:33,891 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:33,891 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=0e38bed8d085aac2e0e9aec5f988def4, regionState=OPEN, openSeqNum=5, regionLocation=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:33,892 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 because future has completed 2024-11-21T11:16:33,894 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=151 2024-11-21T11:16:33,894 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=151, state=SUCCESS, hasLock=false; OpenRegionProcedure 0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 in 168 msec 2024-11-21T11:16:33,895 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-21T11:16:33,895 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e38bed8d085aac2e0e9aec5f988def4, REOPEN/MOVE in 485 msec 2024-11-21T11:16:33,897 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-21T11:16:33,897 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 488 msec 2024-11-21T11:16:33,898 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 901 msec 2024-11-21T11:16:33,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-21T11:16:33,899 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:16:33,900 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e83be8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:16:33,900 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:16:33,900 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:16:33,901 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:16:33,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:16:33,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:16:33,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ffedb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:16:33,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:16:33,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:16:33,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:33,902 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:16:33,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f64fc3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:16:33,903 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c8d9ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:16:33,903 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:16:33,903 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:16:33,904 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:16:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:16:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:16:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b7d0849, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:16:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:16:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:16:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:33,905 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36824, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:16:33,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fa419cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:16:33,906 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f0cd5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:16:33,906 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:16:33,906 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:16:33,906 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:16:33,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:16:33,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:16:33,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dbcc3c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:16:33,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:16:33,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:16:33,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:33,907 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:16:33,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d01482d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:16:33,908 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99b73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:16:33,908 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:16:33,908 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:16:33,909 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:16:33,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:16:33,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:16:33,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17304ec8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:16:33,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:16:33,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:16:33,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:33,910 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36864, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:16:33,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f974138, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:16:33,911 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af69a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:16:33,911 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:16:33,911 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:16:33,911 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:16:33,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:16:33,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:16:33,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57c61d93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:16:33,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:16:33,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:16:33,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:33,912 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:16:33,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24ef9fea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:16:33,913 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb7cff8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:16:33,913 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:16:33,913 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:16:33,914 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:16:33,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:16:33,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:16:33,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3365cb41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:16:33,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:16:33,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:16:33,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:33,915 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:16:33,915 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e25610f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:16:33,916 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ae597b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:16:33,916 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:16:33,916 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:16:33,916 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:16:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:16:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:16:33,917 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2103660d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:16:33,917 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:16:33,917 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:16:33,917 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:33,917 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:16:33,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@782b7ba7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:16:33,918 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39174ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:16:33,918 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:16:33,918 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:16:33,919 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:16:33,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:16:33,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:16:33,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1247b212, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:16:33,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:16:33,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:16:33,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:33,920 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:16:33,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13fde8d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:16:33,921 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8fc71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:16:33,921 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:16:33,921 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:16:33,921 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:16:33,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:16:33,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:16:33,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2969fdac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:16:33,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:16:33,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:16:33,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:33,923 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:16:33,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fd92e50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:16:33,923 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dc96918, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:16:33,923 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,36543,-1 for getting cluster id 2024-11-21T11:16:33,924 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:16:33,924 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '639acd60-369b-4575-8578-7b7e510d276f' 2024-11-21T11:16:33,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:16:33,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "639acd60-369b-4575-8578-7b7e510d276f" 2024-11-21T11:16:33,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@282d52dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:16:33,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,36543,-1] 2024-11-21T11:16:33,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:16:33,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:33,925 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:16:33,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42975032, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:16:33,926 DEBUG [Thread-2489 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:16:33,926 DEBUG [Thread-2478 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:16:33,926 DEBUG [Thread-2474 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:16:33,926 DEBUG [Thread-2485 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:16:33,927 DEBUG [Thread-2470 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:16:33,927 DEBUG [Thread-2476 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:16:33,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:16:33,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:16:33,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:16:33,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:16:33,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:16:33,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:16:33,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:16:33,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:16:33,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:16:33,927 DEBUG [Thread-2481 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:16:33,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:16:33,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:16:33,927 DEBUG [Thread-2487 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:16:33,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:16:33,928 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34262, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:16:33,928 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:16:33,928 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34274, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:16:33,929 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:16:33,929 DEBUG [Thread-2483 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:16:33,929 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:16:33,929 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:16:33,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:16:33,929 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34294, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:16:33,929 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:16:33,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:16:33,929 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:16:33,929 DEBUG [Thread-2472 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:16:33,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:16:33,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5] 2024-11-21T11:16:33,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:16:33,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34320, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:16:33,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:16:33,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5] 2024-11-21T11:16:33,931 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5] 2024-11-21T11:16:33,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,33417,1732187618276, seqNum=-1] 2024-11-21T11:16:33,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:16:33,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5] 2024-11-21T11:16:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5] 2024-11-21T11:16:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-21T11:16:33,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5] 2024-11-21T11:16:33,933 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5] 2024-11-21T11:16:33,933 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:16:33,933 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:16:33,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5] 2024-11-21T11:16:33,934 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:16:33,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T11:16:33,935 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5] 2024-11-21T11:16:33,935 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:16:33,935 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5] 2024-11-21T11:16:33,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:16:33,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:33,938 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0e38bed8d085aac2e0e9aec5f988def4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T11:16:33,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=A 2024-11-21T11:16:33,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:33,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=B 2024-11-21T11:16:33,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:33,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=C 2024-11-21T11:16:33,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:33,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:33,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187853956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:33,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732187793945, max timestamp=1732187793959Num uniques 859; 2024-11-21T11:16:33,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T11:16:33,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732187793945, max timestamp=1732187793959Num uniques 859; 2024-11-21T11:16:33,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T11:16:33,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732187793945, max timestamp=1732187793959Num uniques 859; 2024-11-21T11:16:33,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T11:16:33,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:33,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:33,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:33,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121bcc10b69d19e4464a046e9d1beb86fcf_0e38bed8d085aac2e0e9aec5f988def4 is 50, key is test_row_0/A:col10/1732187793938/Put/seqid=0 2024-11-21T11:16:33,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742378_1554 (size=14594) 2024-11-21T11:16:33,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:33,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187853981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:33,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:33,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187853982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:33,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:33,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:33,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:33,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:33,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:33,987 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:33,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:33,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187853984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:33,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:33,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:33,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:33,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187853984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:33,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:33,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:33,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:33,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:34,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T11:16:34,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:34,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187854066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:34,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:34,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:34,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:34,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T11:16:34,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:34,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:34,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:34,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:34,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:34,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:34,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:34,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187854095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:34,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:34,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:34,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:34,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187854095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:34,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:34,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:34,097 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:34,097 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:34,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187854095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:34,097 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:34,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:34,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:34,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:34,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:34,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187854096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:34,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:34,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:34,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:34,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T11:16:34,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:34,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:34,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:34,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:34,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:34,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:34,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T11:16:34,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:34,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187854275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:34,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:34,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:34,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:34,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:34,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187854305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:34,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:34,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:34,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:34,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187854305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:34,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:34,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:34,310 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:34,310 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:34,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187854305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:34,310 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:34,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:34,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:34,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:34,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:34,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187854305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:34,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:34,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:34,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:34,381 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:34,385 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121bcc10b69d19e4464a046e9d1beb86fcf_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121bcc10b69d19e4464a046e9d1beb86fcf_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:34,385 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/f7676d8f639c49439340258dea457155, store: [table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:34,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/f7676d8f639c49439340258dea457155 is 175, key is test_row_0/A:col10/1732187793938/Put/seqid=0 2024-11-21T11:16:34,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T11:16:34,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:34,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:34,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:34,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:34,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:34,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:34,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742379_1555 (size=39549) 2024-11-21T11:16:34,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T11:16:34,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:34,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:34,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:34,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:34,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:34,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:34,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T11:16:34,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:34,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187854586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:34,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:34,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:34,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:34,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:34,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187854615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:34,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:34,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:34,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:34,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187854615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:34,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:34,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:34,617 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:34,617 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:34,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187854615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:34,617 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:34,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:34,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:34,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:34,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187854615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:34,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:34,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:34,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:34,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:34,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T11:16:34,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:34,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:34,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:34,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:34,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:34,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:34,796 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/f7676d8f639c49439340258dea457155 2024-11-21T11:16:34,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/44d561c4d7944e13809625c667ee8b2d is 50, key is test_row_0/B:col10/1732187793938/Put/seqid=0 2024-11-21T11:16:34,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742380_1556 (size=12001) 2024-11-21T11:16:34,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T11:16:34,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:34,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:34,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:34,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:34,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:35,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T11:16:35,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:35,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:35,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:35,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:35,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:35,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:35,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T11:16:35,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:35,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187855095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:35,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:35,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:35,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:35,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:35,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187855125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:35,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:35,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:35,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:35,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:35,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187855125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:35,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:35,130 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:35,130 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:35,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187855125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:35,130 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:35,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:35,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:35,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187855126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:35,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:35,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:35,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:35,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:35,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:35,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T11:16:35,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:35,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:35,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:35,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:35,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:35,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:35,220 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/44d561c4d7944e13809625c667ee8b2d 2024-11-21T11:16:35,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/373ef2f33e6a4d15920c9d65c609d784 is 50, key is test_row_0/C:col10/1732187793938/Put/seqid=0 2024-11-21T11:16:35,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742381_1557 (size=12001) 2024-11-21T11:16:35,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/373ef2f33e6a4d15920c9d65c609d784 2024-11-21T11:16:35,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/f7676d8f639c49439340258dea457155 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/f7676d8f639c49439340258dea457155 2024-11-21T11:16:35,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/f7676d8f639c49439340258dea457155, entries=200, sequenceid=15, filesize=38.6 K 2024-11-21T11:16:35,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/44d561c4d7944e13809625c667ee8b2d as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/44d561c4d7944e13809625c667ee8b2d 2024-11-21T11:16:35,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/44d561c4d7944e13809625c667ee8b2d, entries=150, sequenceid=15, filesize=11.7 K 2024-11-21T11:16:35,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/373ef2f33e6a4d15920c9d65c609d784 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/373ef2f33e6a4d15920c9d65c609d784 2024-11-21T11:16:35,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/373ef2f33e6a4d15920c9d65c609d784, entries=150, sequenceid=15, filesize=11.7 K 2024-11-21T11:16:35,264 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 0e38bed8d085aac2e0e9aec5f988def4 in 1326ms, sequenceid=15, compaction requested=false 2024-11-21T11:16:35,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0e38bed8d085aac2e0e9aec5f988def4: 2024-11-21T11:16:35,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T11:16:35,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:35,311 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 0e38bed8d085aac2e0e9aec5f988def4 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-21T11:16:35,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=A 2024-11-21T11:16:35,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:16:35,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=B 2024-11-21T11:16:35,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:16:35,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=C 2024-11-21T11:16:35,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:16:35,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121cf8791b4036549deb318846207ed3994_0e38bed8d085aac2e0e9aec5f988def4 is 50, key is test_row_0/A:col10/1732187793982/Put/seqid=0 2024-11-21T11:16:35,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742382_1558 (size=12154) 2024-11-21T11:16:35,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,326 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121cf8791b4036549deb318846207ed3994_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121cf8791b4036549deb318846207ed3994_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:35,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/50c3b10cfe914b9b92e4528dce2f3947, store: [table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:35,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/50c3b10cfe914b9b92e4528dce2f3947 is 175, key is test_row_0/A:col10/1732187793982/Put/seqid=0 2024-11-21T11:16:35,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742383_1559 (size=30955) 2024-11-21T11:16:35,340 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/50c3b10cfe914b9b92e4528dce2f3947 2024-11-21T11:16:35,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/83dff377048747ee97598048ad6301ca is 50, key is test_row_0/B:col10/1732187793982/Put/seqid=0 2024-11-21T11:16:35,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742384_1560 (size=12001) 2024-11-21T11:16:35,355 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/83dff377048747ee97598048ad6301ca 2024-11-21T11:16:35,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/4d9e7495876b4908a35483865a54b90b is 50, key is test_row_0/C:col10/1732187793982/Put/seqid=0 2024-11-21T11:16:35,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742385_1561 (size=12001) 2024-11-21T11:16:35,373 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/4d9e7495876b4908a35483865a54b90b 2024-11-21T11:16:35,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/50c3b10cfe914b9b92e4528dce2f3947 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/50c3b10cfe914b9b92e4528dce2f3947 2024-11-21T11:16:35,382 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/50c3b10cfe914b9b92e4528dce2f3947, entries=150, sequenceid=50, filesize=30.2 K 2024-11-21T11:16:35,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/83dff377048747ee97598048ad6301ca as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/83dff377048747ee97598048ad6301ca 2024-11-21T11:16:35,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,386 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/83dff377048747ee97598048ad6301ca, entries=150, sequenceid=50, filesize=11.7 K 2024-11-21T11:16:35,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/4d9e7495876b4908a35483865a54b90b as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/4d9e7495876b4908a35483865a54b90b 2024-11-21T11:16:35,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,390 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/4d9e7495876b4908a35483865a54b90b, entries=150, sequenceid=50, filesize=11.7 K 2024-11-21T11:16:35,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,391 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=0 B/0 for 0e38bed8d085aac2e0e9aec5f988def4 in 80ms, sequenceid=50, compaction requested=false 2024-11-21T11:16:35,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 0e38bed8d085aac2e0e9aec5f988def4: 2024-11-21T11:16:35,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:35,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-21T11:16:35,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-21T11:16:35,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-21T11:16:35,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4570 sec 2024-11-21T11:16:35,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.4660 sec 2024-11-21T11:16:35,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,552 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T11:16:35,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:35,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T11:16:36,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,086 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:16:36,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,086 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:16:36,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-21T11:16:36,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T11:16:36,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,089 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:16:36,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,090 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:16:36,090 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:16:36,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:36,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0e38bed8d085aac2e0e9aec5f988def4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T11:16:36,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=A 2024-11-21T11:16:36,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:36,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=B 2024-11-21T11:16:36,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:36,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=C 2024-11-21T11:16:36,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:36,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T11:16:36,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112103c235dcaf50444a82b34a3bfeded90c_0e38bed8d085aac2e0e9aec5f988def4 is 50, key is test_row_0/A:col10/1732187796173/Put/seqid=0 2024-11-21T11:16:36,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:36,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187856210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:36,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732187796202, max timestamp=1732187796213Num uniques 859; 2024-11-21T11:16:36,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T11:16:36,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732187796202, max timestamp=1732187796213Num uniques 859; 2024-11-21T11:16:36,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-21T11:16:36,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732187796202, max timestamp=1732187796213Num uniques 859; 2024-11-21T11:16:36,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-21T11:16:36,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:36,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742387_1563 (size=24358) 2024-11-21T11:16:36,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T11:16:36,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:36,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:36,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:36,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:36,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:36,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:36,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:36,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187856239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:36,245 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,245 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,245 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:36,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:36,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187856240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:36,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:36,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187856241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:36,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:36,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:36,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:36,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187856245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:36,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:36,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:36,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187856325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:36,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:36,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:36,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187856355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:36,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:36,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187856355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:36,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:36,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:36,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187856355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:36,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:36,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:36,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:36,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187856355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:36,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:36,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T11:16:36,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:36,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:36,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:36,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:36,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:36,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:36,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T11:16:36,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:36,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187856535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:36,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:36,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T11:16:36,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:36,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:36,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:36,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:36,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:36,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:36,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:36,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187856565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:36,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:36,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187856565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:36,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:36,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:36,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187856566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:36,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:36,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:36,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187856566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:36,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:36,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:36,627 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:36,631 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112103c235dcaf50444a82b34a3bfeded90c_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112103c235dcaf50444a82b34a3bfeded90c_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:36,632 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/fea010b43add4d819c0c4e21e90e1021, store: [table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:36,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/fea010b43add4d819c0c4e21e90e1021 is 175, key is test_row_0/A:col10/1732187796173/Put/seqid=0 2024-11-21T11:16:36,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742386_1562 (size=73995) 2024-11-21T11:16:36,657 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T11:16:36,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T11:16:36,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:36,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:36,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:36,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:36,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:36,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:36,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T11:16:36,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:36,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187856845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:36,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:36,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T11:16:36,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:36,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:36,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:36,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:36,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:36,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:36,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:36,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187856876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:36,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:36,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187856876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:36,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:36,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:36,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187856876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:36,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:36,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:36,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187856876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:36,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:36,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:36,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:37,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T11:16:37,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:37,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:37,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:37,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:37,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:37,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:37,035 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/fea010b43add4d819c0c4e21e90e1021 2024-11-21T11:16:37,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/dc2ce56410f149ec948c0b3d710cfba3 is 50, key is test_row_0/B:col10/1732187796173/Put/seqid=0 2024-11-21T11:16:37,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742388_1564 (size=12001) 2024-11-21T11:16:37,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T11:16:37,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:37,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:37,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:37,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:37,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:37,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:37,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T11:16:37,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T11:16:37,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:37,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:37,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:37,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:37,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:37,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:37,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187857356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:37,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:37,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:37,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:37,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:37,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187857385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:37,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:37,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187857385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:37,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:37,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:37,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:37,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:37,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:37,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:37,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:37,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187857385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:37,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:37,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:37,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:37,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:37,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187857385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:37,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:37,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:37,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:37,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/dc2ce56410f149ec948c0b3d710cfba3 2024-11-21T11:16:37,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/869e208af5e54da6a0aa1bcd15e7e3d7 is 50, key is test_row_0/C:col10/1732187796173/Put/seqid=0 2024-11-21T11:16:37,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742389_1565 (size=12001) 2024-11-21T11:16:37,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T11:16:37,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:37,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:37,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:37,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:37,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:37,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:37,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T11:16:37,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:37,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:37,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:37,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:37,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:37,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:37,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T11:16:37,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:37,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:37,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:37,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:37,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:37,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:37,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/869e208af5e54da6a0aa1bcd15e7e3d7 2024-11-21T11:16:37,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/fea010b43add4d819c0c4e21e90e1021 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/fea010b43add4d819c0c4e21e90e1021 2024-11-21T11:16:37,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/fea010b43add4d819c0c4e21e90e1021, entries=400, sequenceid=61, filesize=72.3 K 2024-11-21T11:16:37,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/dc2ce56410f149ec948c0b3d710cfba3 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/dc2ce56410f149ec948c0b3d710cfba3 2024-11-21T11:16:37,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/dc2ce56410f149ec948c0b3d710cfba3, entries=150, sequenceid=61, filesize=11.7 K 2024-11-21T11:16:37,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/869e208af5e54da6a0aa1bcd15e7e3d7 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/869e208af5e54da6a0aa1bcd15e7e3d7 2024-11-21T11:16:37,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/869e208af5e54da6a0aa1bcd15e7e3d7, entries=150, sequenceid=61, filesize=11.7 K 2024-11-21T11:16:37,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=124.44 KB/127430 for 0e38bed8d085aac2e0e9aec5f988def4 in 1702ms, sequenceid=61, compaction requested=true 2024-11-21T11:16:37,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0e38bed8d085aac2e0e9aec5f988def4: 2024-11-21T11:16:37,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e38bed8d085aac2e0e9aec5f988def4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:16:37,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:37,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e38bed8d085aac2e0e9aec5f988def4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:16:37,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:37,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e38bed8d085aac2e0e9aec5f988def4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:16:37,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:16:37,878 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:16:37,878 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:16:37,879 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 144499 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:16:37,879 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 0e38bed8d085aac2e0e9aec5f988def4/A is initiating minor compaction (all files) 2024-11-21T11:16:37,879 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e38bed8d085aac2e0e9aec5f988def4/A in TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:37,879 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/f7676d8f639c49439340258dea457155, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/50c3b10cfe914b9b92e4528dce2f3947, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/fea010b43add4d819c0c4e21e90e1021] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp, totalSize=141.1 K 2024-11-21T11:16:37,879 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:37,879 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. files: [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/f7676d8f639c49439340258dea457155, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/50c3b10cfe914b9b92e4528dce2f3947, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/fea010b43add4d819c0c4e21e90e1021] 2024-11-21T11:16:37,879 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:16:37,879 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 0e38bed8d085aac2e0e9aec5f988def4/B is initiating minor compaction (all files) 2024-11-21T11:16:37,879 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e38bed8d085aac2e0e9aec5f988def4/B in TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:37,879 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7676d8f639c49439340258dea457155, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732187793933 2024-11-21T11:16:37,879 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/44d561c4d7944e13809625c667ee8b2d, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/83dff377048747ee97598048ad6301ca, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/dc2ce56410f149ec948c0b3d710cfba3] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp, totalSize=35.2 K 2024-11-21T11:16:37,880 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50c3b10cfe914b9b92e4528dce2f3947, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732187793970 2024-11-21T11:16:37,880 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 44d561c4d7944e13809625c667ee8b2d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732187793937 2024-11-21T11:16:37,880 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting fea010b43add4d819c0c4e21e90e1021, keycount=400, bloomtype=ROW, size=72.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732187796133 2024-11-21T11:16:37,880 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 83dff377048747ee97598048ad6301ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732187793970 2024-11-21T11:16:37,880 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting dc2ce56410f149ec948c0b3d710cfba3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732187796167 2024-11-21T11:16:37,888 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:37,890 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e38bed8d085aac2e0e9aec5f988def4#B#compaction#480 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:37,891 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/d19bc586a30b4b208747529bc1120ad9 is 50, key is test_row_0/B:col10/1732187796173/Put/seqid=0 2024-11-21T11:16:37,899 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112187c6f3c1a26d423db956ddfec4c3e055_0e38bed8d085aac2e0e9aec5f988def4 store=[table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:37,902 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112187c6f3c1a26d423db956ddfec4c3e055_0e38bed8d085aac2e0e9aec5f988def4, store=[table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:37,903 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112187c6f3c1a26d423db956ddfec4c3e055_0e38bed8d085aac2e0e9aec5f988def4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:37,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742390_1566 (size=12104) 2024-11-21T11:16:37,918 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/d19bc586a30b4b208747529bc1120ad9 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/d19bc586a30b4b208747529bc1120ad9 2024-11-21T11:16:37,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742391_1567 (size=4469) 2024-11-21T11:16:37,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T11:16:37,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:37,924 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0e38bed8d085aac2e0e9aec5f988def4/B of 0e38bed8d085aac2e0e9aec5f988def4 into d19bc586a30b4b208747529bc1120ad9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:37,924 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 0e38bed8d085aac2e0e9aec5f988def4 3/3 column families, dataSize=124.44 KB heapSize=353.47 KB 2024-11-21T11:16:37,924 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e38bed8d085aac2e0e9aec5f988def4: 2024-11-21T11:16:37,924 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., storeName=0e38bed8d085aac2e0e9aec5f988def4/B, priority=13, startTime=1732187797878; duration=0sec 2024-11-21T11:16:37,924 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:16:37,924 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e38bed8d085aac2e0e9aec5f988def4:B 2024-11-21T11:16:37,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=A 2024-11-21T11:16:37,924 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:16:37,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:16:37,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=B 2024-11-21T11:16:37,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:16:37,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=C 2024-11-21T11:16:37,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:16:37,925 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:16:37,925 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 0e38bed8d085aac2e0e9aec5f988def4/C is initiating minor compaction (all files) 2024-11-21T11:16:37,925 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e38bed8d085aac2e0e9aec5f988def4/C in TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:37,925 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/373ef2f33e6a4d15920c9d65c609d784, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/4d9e7495876b4908a35483865a54b90b, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/869e208af5e54da6a0aa1bcd15e7e3d7] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp, totalSize=35.2 K 2024-11-21T11:16:37,925 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 373ef2f33e6a4d15920c9d65c609d784, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732187793937 2024-11-21T11:16:37,927 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d9e7495876b4908a35483865a54b90b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732187793970 2024-11-21T11:16:37,927 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 869e208af5e54da6a0aa1bcd15e7e3d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732187796167 2024-11-21T11:16:37,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112111e6746aef634344b96c8060d3693320_0e38bed8d085aac2e0e9aec5f988def4 is 50, key is test_row_0/A:col10/1732187796238/Put/seqid=0 2024-11-21T11:16:37,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742392_1568 (size=12154) 2024-11-21T11:16:37,947 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e38bed8d085aac2e0e9aec5f988def4#C#compaction#482 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:37,947 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/195f8a72eefb45a9983143066151bbfb is 50, key is test_row_0/C:col10/1732187796173/Put/seqid=0 2024-11-21T11:16:37,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742393_1569 (size=12104) 2024-11-21T11:16:37,972 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/195f8a72eefb45a9983143066151bbfb as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/195f8a72eefb45a9983143066151bbfb 2024-11-21T11:16:37,976 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0e38bed8d085aac2e0e9aec5f988def4/C of 0e38bed8d085aac2e0e9aec5f988def4 into 195f8a72eefb45a9983143066151bbfb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:37,976 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e38bed8d085aac2e0e9aec5f988def4: 2024-11-21T11:16:37,976 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., storeName=0e38bed8d085aac2e0e9aec5f988def4/C, priority=13, startTime=1732187797878; duration=0sec 2024-11-21T11:16:37,976 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:37,976 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e38bed8d085aac2e0e9aec5f988def4:C 2024-11-21T11:16:38,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T11:16:38,324 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e38bed8d085aac2e0e9aec5f988def4#A#compaction#479 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:38,325 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/94421763ba6746558071a1eef8a45a88 is 175, key is test_row_0/A:col10/1732187796173/Put/seqid=0 2024-11-21T11:16:38,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742394_1570 (size=31058) 2024-11-21T11:16:38,333 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/94421763ba6746558071a1eef8a45a88 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/94421763ba6746558071a1eef8a45a88 2024-11-21T11:16:38,339 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0e38bed8d085aac2e0e9aec5f988def4/A of 0e38bed8d085aac2e0e9aec5f988def4 into 94421763ba6746558071a1eef8a45a88(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:38,339 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e38bed8d085aac2e0e9aec5f988def4: 2024-11-21T11:16:38,339 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., storeName=0e38bed8d085aac2e0e9aec5f988def4/A, priority=13, startTime=1732187797878; duration=0sec 2024-11-21T11:16:38,339 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:38,339 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e38bed8d085aac2e0e9aec5f988def4:A 2024-11-21T11:16:38,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:38,345 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112111e6746aef634344b96c8060d3693320_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112111e6746aef634344b96c8060d3693320_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:38,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/e05aacc551db48fcb610950db41e8d7e, store: [table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:38,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/e05aacc551db48fcb610950db41e8d7e is 175, key is test_row_0/A:col10/1732187796238/Put/seqid=0 2024-11-21T11:16:38,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742395_1571 (size=30955) 2024-11-21T11:16:38,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:38,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:38,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:38,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187858404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:38,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:38,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:38,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:38,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187858404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:38,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:38,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187858405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:38,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:38,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:38,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:38,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:38,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:38,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187858405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:38,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:38,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187858405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:38,410 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_0' in 0e38bed8d085aac2e0e9aec5f988def4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:16:38,411 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_0' in 0e38bed8d085aac2e0e9aec5f988def4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:16:38,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:38,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:38,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:38,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:38,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:38,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:38,411 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_0' in 0e38bed8d085aac2e0e9aec5f988def4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:16:38,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:38,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:38,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:38,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:38,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187858515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:38,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:38,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187858516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:38,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:38,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:38,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:38,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:38,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:38,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:38,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187858725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:38,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:38,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:38,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:38,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:38,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187858725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:38,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:38,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:38,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:38,751 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/e05aacc551db48fcb610950db41e8d7e 2024-11-21T11:16:38,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/5f6f7e002841472eab344f4f992f0591 is 50, key is test_row_0/B:col10/1732187796238/Put/seqid=0 2024-11-21T11:16:38,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742396_1572 (size=12001) 2024-11-21T11:16:39,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:39,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187859036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:39,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:39,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187859036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:39,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:39,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:39,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:39,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:39,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:39,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:39,161 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.86 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/5f6f7e002841472eab344f4f992f0591 2024-11-21T11:16:39,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/5fa1a0aa872d49cb91b0158c956463e4 is 50, key is test_row_0/C:col10/1732187796238/Put/seqid=0 2024-11-21T11:16:39,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742397_1573 (size=12001) 2024-11-21T11:16:39,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:39,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187859545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:39,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:39,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:39,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187859546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:39,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:39,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:39,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:39,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:39,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:39,572 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.86 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/5fa1a0aa872d49cb91b0158c956463e4 2024-11-21T11:16:39,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/e05aacc551db48fcb610950db41e8d7e as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/e05aacc551db48fcb610950db41e8d7e 2024-11-21T11:16:39,579 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/e05aacc551db48fcb610950db41e8d7e, entries=150, sequenceid=97, filesize=30.2 K 2024-11-21T11:16:39,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/5f6f7e002841472eab344f4f992f0591 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/5f6f7e002841472eab344f4f992f0591 2024-11-21T11:16:39,583 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/5f6f7e002841472eab344f4f992f0591, entries=150, sequenceid=97, filesize=11.7 K 2024-11-21T11:16:39,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/5fa1a0aa872d49cb91b0158c956463e4 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/5fa1a0aa872d49cb91b0158c956463e4 2024-11-21T11:16:39,586 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/5fa1a0aa872d49cb91b0158c956463e4, entries=150, sequenceid=97, filesize=11.7 K 2024-11-21T11:16:39,587 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~124.44 KB/127430, heapSize ~353.42 KB/361904, currentSize=53.67 KB/54960 for 0e38bed8d085aac2e0e9aec5f988def4 in 1664ms, sequenceid=97, compaction requested=false 2024-11-21T11:16:39,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 0e38bed8d085aac2e0e9aec5f988def4: 2024-11-21T11:16:39,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:39,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-21T11:16:39,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-21T11:16:39,590 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-21T11:16:39,590 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4990 sec 2024-11-21T11:16:39,592 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 3.5050 sec 2024-11-21T11:16:40,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T11:16:40,246 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:16:40,246 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:16:40,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-21T11:16:40,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-21T11:16:40,248 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:16:40,248 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:16:40,249 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:16:40,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-21T11:16:40,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-21T11:16:40,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:40,401 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 0e38bed8d085aac2e0e9aec5f988def4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-21T11:16:40,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=A 2024-11-21T11:16:40,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:40,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=B 2024-11-21T11:16:40,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:40,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=C 2024-11-21T11:16:40,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:40,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112126bfa94334fb4c99a79c3a08dcb02213_0e38bed8d085aac2e0e9aec5f988def4 is 50, key is test_row_0/A:col10/1732187798403/Put/seqid=0 2024-11-21T11:16:40,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742398_1574 (size=12154) 2024-11-21T11:16:40,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:40,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:40,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:40,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187860450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:40,456 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:40,456 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:40,456 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:40,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732187800442, max timestamp=1732187800454Num uniques 859; 2024-11-21T11:16:40,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T11:16:40,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732187800442, max timestamp=1732187800454Num uniques 859; 2024-11-21T11:16:40,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T11:16:40,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732187800442, max timestamp=1732187800454Num uniques 859; 2024-11-21T11:16:40,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T11:16:40,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:40,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187860497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:40,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:40,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:40,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:40,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:40,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187860503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:40,509 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:40,509 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:40,509 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:40,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:40,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187860555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:40,560 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_2' in 0e38bed8d085aac2e0e9aec5f988def4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:16:40,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:40,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:40,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:40,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-21T11:16:40,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:40,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187860566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:40,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:40,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187860566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:40,570 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_1' in 0e38bed8d085aac2e0e9aec5f988def4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:16:40,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:40,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:40,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:40,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:40,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:40,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:40,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:40,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187860605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:40,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:40,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:40,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:40,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:40,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187860615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:40,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:40,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:40,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:40,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:40,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187860775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:40,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:40,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:40,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:40,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:40,815 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112126bfa94334fb4c99a79c3a08dcb02213_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112126bfa94334fb4c99a79c3a08dcb02213_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:40,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/71bfeadd2535442793ce652c07380b04, store: [table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:40,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/71bfeadd2535442793ce652c07380b04 is 175, key is test_row_0/A:col10/1732187798403/Put/seqid=0 2024-11-21T11:16:40,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:40,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187860815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:40,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:40,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:40,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:40,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742399_1575 (size=30955) 2024-11-21T11:16:40,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:40,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187860825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:40,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:40,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:40,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:40,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-21T11:16:41,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:41,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187861085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:41,090 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:41,090 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:41,090 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:41,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:41,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187861125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:41,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:41,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:41,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:41,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:41,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187861135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:41,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:41,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:41,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:41,221 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=110, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/71bfeadd2535442793ce652c07380b04 2024-11-21T11:16:41,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/74b31c39f2a94319a84fe056ddb15975 is 50, key is test_row_0/B:col10/1732187798403/Put/seqid=0 2024-11-21T11:16:41,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742400_1576 (size=12001) 2024-11-21T11:16:41,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-21T11:16:41,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:41,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187861596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:41,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:41,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:41,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:41,632 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/74b31c39f2a94319a84fe056ddb15975 2024-11-21T11:16:41,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/dceab80b7bab445db550370536996849 is 50, key is test_row_0/C:col10/1732187798403/Put/seqid=0 2024-11-21T11:16:41,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:41,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187861635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:41,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:41,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:41,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:41,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742401_1577 (size=12001) 2024-11-21T11:16:41,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:41,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187861645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:41,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:41,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:41,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:42,042 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/dceab80b7bab445db550370536996849 2024-11-21T11:16:42,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/71bfeadd2535442793ce652c07380b04 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/71bfeadd2535442793ce652c07380b04 2024-11-21T11:16:42,049 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/71bfeadd2535442793ce652c07380b04, entries=150, sequenceid=110, filesize=30.2 K 2024-11-21T11:16:42,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/74b31c39f2a94319a84fe056ddb15975 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/74b31c39f2a94319a84fe056ddb15975 2024-11-21T11:16:42,055 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/74b31c39f2a94319a84fe056ddb15975, entries=150, sequenceid=110, filesize=11.7 K 2024-11-21T11:16:42,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/dceab80b7bab445db550370536996849 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/dceab80b7bab445db550370536996849 2024-11-21T11:16:42,059 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/dceab80b7bab445db550370536996849, entries=150, sequenceid=110, filesize=11.7 K 2024-11-21T11:16:42,061 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 0e38bed8d085aac2e0e9aec5f988def4 in 1660ms, sequenceid=110, compaction requested=true 2024-11-21T11:16:42,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 0e38bed8d085aac2e0e9aec5f988def4: 2024-11-21T11:16:42,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:42,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-21T11:16:42,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-21T11:16:42,064 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-21T11:16:42,064 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8140 sec 2024-11-21T11:16:42,066 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.8180 sec 2024-11-21T11:16:42,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-21T11:16:42,395 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:16:42,395 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:16:42,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-21T11:16:42,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T11:16:42,397 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:16:42,398 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:16:42,398 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:16:42,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T11:16:42,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T11:16:42,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:42,551 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 0e38bed8d085aac2e0e9aec5f988def4 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-21T11:16:42,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=A 2024-11-21T11:16:42,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:16:42,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=B 2024-11-21T11:16:42,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:16:42,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=C 2024-11-21T11:16:42,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:16:42,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112147a6953e06a74192a168bf470dbfca30_0e38bed8d085aac2e0e9aec5f988def4 is 50, key is test_row_0/A:col10/1732187800487/Put/seqid=0 2024-11-21T11:16:42,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742402_1578 (size=12304) 2024-11-21T11:16:42,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:42,568 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112147a6953e06a74192a168bf470dbfca30_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112147a6953e06a74192a168bf470dbfca30_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:42,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/4f54bacd80524a33ab8fd0234cf7ab38, store: [table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:42,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/4f54bacd80524a33ab8fd0234cf7ab38 is 175, key is test_row_0/A:col10/1732187800487/Put/seqid=0 2024-11-21T11:16:42,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:42,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:42,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742403_1579 (size=31105) 2024-11-21T11:16:42,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:42,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187862607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:42,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:42,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:42,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:42,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:42,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187862612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:42,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:42,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:42,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187862612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:42,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:42,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:42,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:42,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:42,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:42,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:42,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187862655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:42,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187862656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:42,658 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_1' in 0e38bed8d085aac2e0e9aec5f988def4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:16:42,658 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_0' in 0e38bed8d085aac2e0e9aec5f988def4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:16:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:42,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T11:16:42,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:42,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187862715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:42,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:42,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:42,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:42,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:42,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187862725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:42,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:42,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:42,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:42,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187862725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:42,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:42,732 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:42,732 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:42,732 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:42,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:42,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187862925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:42,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:42,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:42,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:42,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:42,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187862935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:42,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:42,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:42,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:42,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:42,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187862935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:42,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:42,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:42,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:42,980 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/4f54bacd80524a33ab8fd0234cf7ab38 2024-11-21T11:16:42,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/a30d2a27b9084e6189f3f2eb053b2df1 is 50, key is test_row_0/B:col10/1732187800487/Put/seqid=0 2024-11-21T11:16:42,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742404_1580 (size=12151) 2024-11-21T11:16:42,991 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/a30d2a27b9084e6189f3f2eb053b2df1 2024-11-21T11:16:42,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/70b7fb64646a47328fbd13c2286bd007 is 50, key is test_row_0/C:col10/1732187800487/Put/seqid=0 2024-11-21T11:16:43,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742405_1581 (size=12151) 2024-11-21T11:16:43,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T11:16:43,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:43,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187863235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:43,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:43,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:43,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:43,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:43,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187863245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:43,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:43,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187863245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:43,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:43,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:43,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:43,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:43,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:43,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:43,409 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/70b7fb64646a47328fbd13c2286bd007 2024-11-21T11:16:43,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/4f54bacd80524a33ab8fd0234cf7ab38 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/4f54bacd80524a33ab8fd0234cf7ab38 2024-11-21T11:16:43,417 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/4f54bacd80524a33ab8fd0234cf7ab38, entries=150, sequenceid=145, filesize=30.4 K 2024-11-21T11:16:43,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/a30d2a27b9084e6189f3f2eb053b2df1 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/a30d2a27b9084e6189f3f2eb053b2df1 2024-11-21T11:16:43,421 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/a30d2a27b9084e6189f3f2eb053b2df1, entries=150, sequenceid=145, filesize=11.9 K 2024-11-21T11:16:43,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/70b7fb64646a47328fbd13c2286bd007 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/70b7fb64646a47328fbd13c2286bd007 2024-11-21T11:16:43,425 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/70b7fb64646a47328fbd13c2286bd007, entries=150, sequenceid=145, filesize=11.9 K 2024-11-21T11:16:43,426 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=60.38 KB/61830 for 0e38bed8d085aac2e0e9aec5f988def4 in 876ms, sequenceid=145, compaction requested=true 2024-11-21T11:16:43,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 0e38bed8d085aac2e0e9aec5f988def4: 2024-11-21T11:16:43,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:43,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-21T11:16:43,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-21T11:16:43,429 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-21T11:16:43,429 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0290 sec 2024-11-21T11:16:43,430 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.0330 sec 2024-11-21T11:16:43,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T11:16:43,535 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:16:43,536 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:16:43,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-21T11:16:43,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T11:16:43,538 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:16:43,538 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:16:43,538 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:16:43,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T11:16:43,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-21T11:16:43,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:43,691 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 0e38bed8d085aac2e0e9aec5f988def4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-21T11:16:43,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=A 2024-11-21T11:16:43,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:43,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=B 2024-11-21T11:16:43,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:43,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=C 2024-11-21T11:16:43,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:43,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214a0d49188d104fb38f48a9551200f8f3_0e38bed8d085aac2e0e9aec5f988def4 is 50, key is test_row_0/A:col10/1732187802611/Put/seqid=0 2024-11-21T11:16:43,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742406_1582 (size=12304) 2024-11-21T11:16:43,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:43,704 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214a0d49188d104fb38f48a9551200f8f3_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214a0d49188d104fb38f48a9551200f8f3_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:43,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/c73cd926542c45f79507c3e70f9ec5e7, store: [table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:43,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/c73cd926542c45f79507c3e70f9ec5e7 is 175, key is test_row_0/A:col10/1732187802611/Put/seqid=0 2024-11-21T11:16:43,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742407_1583 (size=31105) 2024-11-21T11:16:43,711 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/c73cd926542c45f79507c3e70f9ec5e7 2024-11-21T11:16:43,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/f1b618043e7c48948e9d482bba5d7e67 is 50, key is test_row_0/B:col10/1732187802611/Put/seqid=0 2024-11-21T11:16:43,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742408_1584 (size=12151) 2024-11-21T11:16:43,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:43,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:43,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187863787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:43,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:43,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:43,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187863788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:43,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:43,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:43,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:43,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187863789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:43,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:43,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:43,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:43,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T11:16:43,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:43,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187863895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:43,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:43,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:43,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:43,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:43,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187863906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:43,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:43,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:43,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:43,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187863906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:43,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:43,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:43,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:43,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:44,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:44,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187864105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:44,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:44,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:44,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:44,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:44,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187864115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:44,120 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/f1b618043e7c48948e9d482bba5d7e67 2024-11-21T11:16:44,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:44,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:44,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:44,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187864116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:44,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:44,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:44,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:44,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:44,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/cdb523f9c4cb4dc59acd9e88913c0c30 is 50, key is test_row_0/C:col10/1732187802611/Put/seqid=0 2024-11-21T11:16:44,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742409_1585 (size=12151) 2024-11-21T11:16:44,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T11:16:44,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:44,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187864416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:44,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:44,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:44,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:44,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:44,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187864425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:44,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:44,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187864425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:44,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:44,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:44,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:44,531 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/cdb523f9c4cb4dc59acd9e88913c0c30 2024-11-21T11:16:44,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/c73cd926542c45f79507c3e70f9ec5e7 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/c73cd926542c45f79507c3e70f9ec5e7 2024-11-21T11:16:44,539 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/c73cd926542c45f79507c3e70f9ec5e7, entries=150, sequenceid=157, filesize=30.4 K 2024-11-21T11:16:44,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/f1b618043e7c48948e9d482bba5d7e67 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/f1b618043e7c48948e9d482bba5d7e67 2024-11-21T11:16:44,543 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/f1b618043e7c48948e9d482bba5d7e67, entries=150, sequenceid=157, filesize=11.9 K 2024-11-21T11:16:44,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/cdb523f9c4cb4dc59acd9e88913c0c30 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/cdb523f9c4cb4dc59acd9e88913c0c30 2024-11-21T11:16:44,546 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/cdb523f9c4cb4dc59acd9e88913c0c30, entries=150, sequenceid=157, filesize=11.9 K 2024-11-21T11:16:44,547 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 0e38bed8d085aac2e0e9aec5f988def4 in 856ms, sequenceid=157, compaction requested=true 2024-11-21T11:16:44,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 0e38bed8d085aac2e0e9aec5f988def4: 2024-11-21T11:16:44,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:44,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-21T11:16:44,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-21T11:16:44,550 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-21T11:16:44,550 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0100 sec 2024-11-21T11:16:44,552 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.0150 sec 2024-11-21T11:16:44,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T11:16:44,676 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:16:44,676 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:16:44,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-21T11:16:44,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T11:16:44,679 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:16:44,679 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:16:44,680 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:16:44,681 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0e38bed8d085aac2e0e9aec5f988def4 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-21T11:16:44,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=A 2024-11-21T11:16:44,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:16:44,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=B 2024-11-21T11:16:44,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:16:44,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=C 2024-11-21T11:16:44,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:16:44,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:44,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214f27d12f66214c379c3ec4e45acf1f54_0e38bed8d085aac2e0e9aec5f988def4 is 50, key is test_row_0/A:col10/1732187803769/Put/seqid=0 2024-11-21T11:16:44,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:44,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187864720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:44,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742410_1586 (size=14794) 2024-11-21T11:16:44,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:44,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:44,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:44,731 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:44,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:44,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187864722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:44,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:44,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:44,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:44,736 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214f27d12f66214c379c3ec4e45acf1f54_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214f27d12f66214c379c3ec4e45acf1f54_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:44,738 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/5091abc444134dd5bf7c6a4af7f94762, store: [table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:44,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/5091abc444134dd5bf7c6a4af7f94762 is 175, key is test_row_0/A:col10/1732187803769/Put/seqid=0 2024-11-21T11:16:44,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742411_1587 (size=39749) 2024-11-21T11:16:44,753 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=178, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/5091abc444134dd5bf7c6a4af7f94762 2024-11-21T11:16:44,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/60223435d4924d1887233c7b58b16193 is 50, key is test_row_0/B:col10/1732187803769/Put/seqid=0 2024-11-21T11:16:44,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T11:16:44,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742412_1588 (size=12151) 2024-11-21T11:16:44,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/60223435d4924d1887233c7b58b16193 2024-11-21T11:16:44,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/4050722995e64a0e8ff88c35286f11a1 is 50, key is test_row_0/C:col10/1732187803769/Put/seqid=0 2024-11-21T11:16:44,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742413_1589 (size=12151) 2024-11-21T11:16:44,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/4050722995e64a0e8ff88c35286f11a1 2024-11-21T11:16:44,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/5091abc444134dd5bf7c6a4af7f94762 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/5091abc444134dd5bf7c6a4af7f94762 2024-11-21T11:16:44,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T11:16:44,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:44,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:44,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:44,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:44,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:44,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:44,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/5091abc444134dd5bf7c6a4af7f94762, entries=200, sequenceid=178, filesize=38.8 K 2024-11-21T11:16:44,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/60223435d4924d1887233c7b58b16193 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/60223435d4924d1887233c7b58b16193 2024-11-21T11:16:44,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:44,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187864836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:44,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:44,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:44,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187864837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:44,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:44,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:44,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:44,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:44,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:44,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/60223435d4924d1887233c7b58b16193, entries=150, sequenceid=178, filesize=11.9 K 2024-11-21T11:16:44,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/4050722995e64a0e8ff88c35286f11a1 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/4050722995e64a0e8ff88c35286f11a1 2024-11-21T11:16:44,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/4050722995e64a0e8ff88c35286f11a1, entries=150, sequenceid=178, filesize=11.9 K 2024-11-21T11:16:44,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 0e38bed8d085aac2e0e9aec5f988def4 in 173ms, sequenceid=178, compaction requested=true 2024-11-21T11:16:44,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0e38bed8d085aac2e0e9aec5f988def4: 2024-11-21T11:16:44,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e38bed8d085aac2e0e9aec5f988def4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:16:44,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:44,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e38bed8d085aac2e0e9aec5f988def4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:16:44,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:16:44,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e38bed8d085aac2e0e9aec5f988def4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:16:44,855 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-21T11:16:44,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T11:16:44,855 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-21T11:16:44,856 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 194927 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-21T11:16:44,857 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 0e38bed8d085aac2e0e9aec5f988def4/A is initiating minor compaction (all files) 2024-11-21T11:16:44,857 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e38bed8d085aac2e0e9aec5f988def4/A in TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:44,857 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/94421763ba6746558071a1eef8a45a88, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/e05aacc551db48fcb610950db41e8d7e, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/71bfeadd2535442793ce652c07380b04, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/4f54bacd80524a33ab8fd0234cf7ab38, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/c73cd926542c45f79507c3e70f9ec5e7, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/5091abc444134dd5bf7c6a4af7f94762] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp, totalSize=190.4 K 2024-11-21T11:16:44,857 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:44,857 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. files: [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/94421763ba6746558071a1eef8a45a88, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/e05aacc551db48fcb610950db41e8d7e, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/71bfeadd2535442793ce652c07380b04, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/4f54bacd80524a33ab8fd0234cf7ab38, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/c73cd926542c45f79507c3e70f9ec5e7, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/5091abc444134dd5bf7c6a4af7f94762] 2024-11-21T11:16:44,859 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72559 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-21T11:16:44,859 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 0e38bed8d085aac2e0e9aec5f988def4/B is initiating minor compaction (all files) 2024-11-21T11:16:44,859 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e38bed8d085aac2e0e9aec5f988def4/B in TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:44,859 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/d19bc586a30b4b208747529bc1120ad9, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/5f6f7e002841472eab344f4f992f0591, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/74b31c39f2a94319a84fe056ddb15975, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/a30d2a27b9084e6189f3f2eb053b2df1, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/f1b618043e7c48948e9d482bba5d7e67, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/60223435d4924d1887233c7b58b16193] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp, totalSize=70.9 K 2024-11-21T11:16:44,859 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94421763ba6746558071a1eef8a45a88, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732187796167 2024-11-21T11:16:44,860 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting e05aacc551db48fcb610950db41e8d7e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732187796235 2024-11-21T11:16:44,860 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting d19bc586a30b4b208747529bc1120ad9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732187796167 2024-11-21T11:16:44,860 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71bfeadd2535442793ce652c07380b04, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732187798396 2024-11-21T11:16:44,860 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f6f7e002841472eab344f4f992f0591, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732187796235 2024-11-21T11:16:44,860 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 74b31c39f2a94319a84fe056ddb15975, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732187798396 2024-11-21T11:16:44,860 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f54bacd80524a33ab8fd0234cf7ab38, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732187800487 2024-11-21T11:16:44,861 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting a30d2a27b9084e6189f3f2eb053b2df1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732187800487 2024-11-21T11:16:44,861 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting c73cd926542c45f79507c3e70f9ec5e7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732187802600 2024-11-21T11:16:44,861 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting f1b618043e7c48948e9d482bba5d7e67, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732187802600 2024-11-21T11:16:44,861 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5091abc444134dd5bf7c6a4af7f94762, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732187803769 2024-11-21T11:16:44,862 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 60223435d4924d1887233c7b58b16193, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732187803769 2024-11-21T11:16:44,879 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:44,893 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e38bed8d085aac2e0e9aec5f988def4#B#compaction#498 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:44,893 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/18a3599b08764dfb9c1d7253cb7dcf38 is 50, key is test_row_0/B:col10/1732187803769/Put/seqid=0 2024-11-21T11:16:44,902 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121638397ff4b1543668f249b982e2e5d20_0e38bed8d085aac2e0e9aec5f988def4 store=[table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:44,908 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121638397ff4b1543668f249b982e2e5d20_0e38bed8d085aac2e0e9aec5f988def4, store=[table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:44,908 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121638397ff4b1543668f249b982e2e5d20_0e38bed8d085aac2e0e9aec5f988def4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:44,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:44,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0e38bed8d085aac2e0e9aec5f988def4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-21T11:16:44,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=A 2024-11-21T11:16:44,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:44,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=B 2024-11-21T11:16:44,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:44,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=C 2024-11-21T11:16:44,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:44,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742414_1590 (size=12459) 2024-11-21T11:16:44,957 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/18a3599b08764dfb9c1d7253cb7dcf38 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/18a3599b08764dfb9c1d7253cb7dcf38 2024-11-21T11:16:44,962 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 0e38bed8d085aac2e0e9aec5f988def4/B of 0e38bed8d085aac2e0e9aec5f988def4 into 18a3599b08764dfb9c1d7253cb7dcf38(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:44,962 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e38bed8d085aac2e0e9aec5f988def4: 2024-11-21T11:16:44,963 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., storeName=0e38bed8d085aac2e0e9aec5f988def4/B, priority=10, startTime=1732187804854; duration=0sec 2024-11-21T11:16:44,963 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:16:44,963 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e38bed8d085aac2e0e9aec5f988def4:B 2024-11-21T11:16:44,963 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-21T11:16:44,964 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72559 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-21T11:16:44,964 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 0e38bed8d085aac2e0e9aec5f988def4/C is initiating minor compaction (all files) 2024-11-21T11:16:44,964 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e38bed8d085aac2e0e9aec5f988def4/C in TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:44,964 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/195f8a72eefb45a9983143066151bbfb, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/5fa1a0aa872d49cb91b0158c956463e4, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/dceab80b7bab445db550370536996849, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/70b7fb64646a47328fbd13c2286bd007, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/cdb523f9c4cb4dc59acd9e88913c0c30, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/4050722995e64a0e8ff88c35286f11a1] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp, totalSize=70.9 K 2024-11-21T11:16:44,965 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 195f8a72eefb45a9983143066151bbfb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732187796167 2024-11-21T11:16:44,965 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fa1a0aa872d49cb91b0158c956463e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732187796235 2024-11-21T11:16:44,965 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting dceab80b7bab445db550370536996849, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732187798396 2024-11-21T11:16:44,965 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 70b7fb64646a47328fbd13c2286bd007, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732187800487 2024-11-21T11:16:44,966 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting cdb523f9c4cb4dc59acd9e88913c0c30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732187802600 2024-11-21T11:16:44,966 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 4050722995e64a0e8ff88c35286f11a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732187803769 2024-11-21T11:16:44,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742415_1591 (size=4469) 2024-11-21T11:16:44,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T11:16:44,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:44,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:44,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:44,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:44,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:44,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:44,991 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e38bed8d085aac2e0e9aec5f988def4#C#compaction#499 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:44,992 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/3630d084e1394c09bc8c02841f65cab1 is 50, key is test_row_0/C:col10/1732187803769/Put/seqid=0 2024-11-21T11:16:44,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T11:16:44,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d30941fae8454ceeaf230c3e4bd8050c_0e38bed8d085aac2e0e9aec5f988def4 is 50, key is test_row_0/A:col10/1732187804692/Put/seqid=0 2024-11-21T11:16:45,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:45,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187865005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:45,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:45,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:45,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:45,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:45,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187865006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:45,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:45,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:45,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:45,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:45,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187865010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:45,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:45,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:45,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:45,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742417_1593 (size=14794) 2024-11-21T11:16:45,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742416_1592 (size=12459) 2024-11-21T11:16:45,049 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/3630d084e1394c09bc8c02841f65cab1 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/3630d084e1394c09bc8c02841f65cab1 2024-11-21T11:16:45,054 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 0e38bed8d085aac2e0e9aec5f988def4/C of 0e38bed8d085aac2e0e9aec5f988def4 into 3630d084e1394c09bc8c02841f65cab1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:45,054 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e38bed8d085aac2e0e9aec5f988def4: 2024-11-21T11:16:45,054 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., storeName=0e38bed8d085aac2e0e9aec5f988def4/C, priority=10, startTime=1732187804855; duration=0sec 2024-11-21T11:16:45,054 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:45,054 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e38bed8d085aac2e0e9aec5f988def4:C 2024-11-21T11:16:45,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:45,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187865056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:45,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:45,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:45,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:45,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:45,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187865056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:45,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:45,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:45,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:45,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:45,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187865125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:45,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:45,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:45,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187865125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:45,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:45,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:45,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:45,130 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:45,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187865125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:45,130 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:45,130 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:45,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:45,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:45,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:45,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T11:16:45,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:45,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:45,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:45,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:45,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:45,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:45,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T11:16:45,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:45,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:45,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:45,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:45,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:45,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:45,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T11:16:45,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:45,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187865335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:45,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:45,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:45,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:45,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:45,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187865335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:45,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:45,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:45,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:45,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187865336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:45,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:45,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:45,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:45,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:45,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:45,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187865365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:45,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:45,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:45,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:45,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187865365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:45,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:45,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:45,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:45,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:45,372 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e38bed8d085aac2e0e9aec5f988def4#A#compaction#497 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:45,372 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/3b1654e6ae974160a5cad36a02c878bf is 175, key is test_row_0/A:col10/1732187803769/Put/seqid=0 2024-11-21T11:16:45,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742418_1594 (size=31413) 2024-11-21T11:16:45,380 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/3b1654e6ae974160a5cad36a02c878bf as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/3b1654e6ae974160a5cad36a02c878bf 2024-11-21T11:16:45,385 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 0e38bed8d085aac2e0e9aec5f988def4/A of 0e38bed8d085aac2e0e9aec5f988def4 into 3b1654e6ae974160a5cad36a02c878bf(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:45,385 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e38bed8d085aac2e0e9aec5f988def4: 2024-11-21T11:16:45,385 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., storeName=0e38bed8d085aac2e0e9aec5f988def4/A, priority=10, startTime=1732187804854; duration=0sec 2024-11-21T11:16:45,385 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:45,385 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e38bed8d085aac2e0e9aec5f988def4:A 2024-11-21T11:16:45,435 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:45,438 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d30941fae8454ceeaf230c3e4bd8050c_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d30941fae8454ceeaf230c3e4bd8050c_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:45,439 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/f3a43e878d904376b737e21c20abdd8f, store: [table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:45,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/f3a43e878d904376b737e21c20abdd8f is 175, key is test_row_0/A:col10/1732187804692/Put/seqid=0 2024-11-21T11:16:45,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742419_1595 (size=39749) 2024-11-21T11:16:45,444 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=190, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/f3a43e878d904376b737e21c20abdd8f 2024-11-21T11:16:45,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T11:16:45,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:45,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:45,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:45,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:45,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:45,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:45,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/3109bb16e2274eeea247aa92f92a3c76 is 50, key is test_row_0/B:col10/1732187804692/Put/seqid=0 2024-11-21T11:16:45,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742420_1596 (size=12151) 2024-11-21T11:16:45,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T11:16:45,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:45,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:45,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:45,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:45,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:45,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:45,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:45,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187865646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:45,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:45,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187865646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:45,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:45,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:45,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:45,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:45,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:45,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187865646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:45,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:45,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:45,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:45,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:45,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:45,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T11:16:45,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:45,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:45,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:45,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:45,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:45,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:45,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T11:16:45,864 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/3109bb16e2274eeea247aa92f92a3c76 2024-11-21T11:16:45,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:45,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187865875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:45,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:45,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:45,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:45,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:45,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187865875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:45,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:45,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:45,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:45,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/bdcc8883224448d78d2ee4798b2877e9 is 50, key is test_row_0/C:col10/1732187804692/Put/seqid=0 2024-11-21T11:16:45,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T11:16:45,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:45,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:45,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:45,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:45,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:45,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:45,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742421_1597 (size=12151) 2024-11-21T11:16:46,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T11:16:46,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:46,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:46,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:46,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:46,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:46,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:46,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:46,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187866156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:46,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:46,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:46,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:46,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187866156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:46,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:46,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:46,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:46,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:46,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:46,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187866156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:46,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:46,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:46,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:46,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T11:16:46,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:46,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:46,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:46,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:46,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:46,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:46,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/bdcc8883224448d78d2ee4798b2877e9 2024-11-21T11:16:46,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/f3a43e878d904376b737e21c20abdd8f as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/f3a43e878d904376b737e21c20abdd8f 2024-11-21T11:16:46,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/f3a43e878d904376b737e21c20abdd8f, entries=200, sequenceid=190, filesize=38.8 K 2024-11-21T11:16:46,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/3109bb16e2274eeea247aa92f92a3c76 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/3109bb16e2274eeea247aa92f92a3c76 2024-11-21T11:16:46,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/3109bb16e2274eeea247aa92f92a3c76, entries=150, sequenceid=190, filesize=11.9 K 2024-11-21T11:16:46,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/bdcc8883224448d78d2ee4798b2877e9 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/bdcc8883224448d78d2ee4798b2877e9 2024-11-21T11:16:46,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/bdcc8883224448d78d2ee4798b2877e9, entries=150, sequenceid=190, filesize=11.9 K 2024-11-21T11:16:46,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T11:16:46,372 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 0e38bed8d085aac2e0e9aec5f988def4 in 1436ms, sequenceid=190, compaction requested=false 2024-11-21T11:16:46,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0e38bed8d085aac2e0e9aec5f988def4: 2024-11-21T11:16:46,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:46,373 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 0e38bed8d085aac2e0e9aec5f988def4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-21T11:16:46,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=A 2024-11-21T11:16:46,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:46,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=B 2024-11-21T11:16:46,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:46,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=C 2024-11-21T11:16:46,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:46,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121144cb6223b8b4b8aad5460ea8257ab84_0e38bed8d085aac2e0e9aec5f988def4 is 50, key is test_row_0/A:col10/1732187804996/Put/seqid=0 2024-11-21T11:16:46,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742422_1598 (size=12304) 2024-11-21T11:16:46,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:46,441 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121144cb6223b8b4b8aad5460ea8257ab84_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121144cb6223b8b4b8aad5460ea8257ab84_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:46,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/ca28eac7eb8a4457897dd0f246cf1dfc, store: [table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:46,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/ca28eac7eb8a4457897dd0f246cf1dfc is 175, key is test_row_0/A:col10/1732187804996/Put/seqid=0 2024-11-21T11:16:46,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742423_1599 (size=31105) 2024-11-21T11:16:46,495 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=213, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/ca28eac7eb8a4457897dd0f246cf1dfc 2024-11-21T11:16:46,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/06e132731563462ba7116efeb25cdbfb is 50, key is test_row_0/B:col10/1732187804996/Put/seqid=0 2024-11-21T11:16:46,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742424_1600 (size=12151) 2024-11-21T11:16:46,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T11:16:46,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:46,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:46,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:46,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187866909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:46,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:46,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:46,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:46,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:46,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187866912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:46,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:46,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:46,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:46,970 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/06e132731563462ba7116efeb25cdbfb 2024-11-21T11:16:46,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/9621f8b90d274258b815c1ef74e7fb0c is 50, key is test_row_0/C:col10/1732187804996/Put/seqid=0 2024-11-21T11:16:46,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742425_1601 (size=12151) 2024-11-21T11:16:46,982 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/9621f8b90d274258b815c1ef74e7fb0c 2024-11-21T11:16:46,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/ca28eac7eb8a4457897dd0f246cf1dfc as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/ca28eac7eb8a4457897dd0f246cf1dfc 2024-11-21T11:16:46,989 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/ca28eac7eb8a4457897dd0f246cf1dfc, entries=150, sequenceid=213, filesize=30.4 K 2024-11-21T11:16:46,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/06e132731563462ba7116efeb25cdbfb as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/06e132731563462ba7116efeb25cdbfb 2024-11-21T11:16:46,993 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/06e132731563462ba7116efeb25cdbfb, entries=150, sequenceid=213, filesize=11.9 K 2024-11-21T11:16:46,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/9621f8b90d274258b815c1ef74e7fb0c as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/9621f8b90d274258b815c1ef74e7fb0c 2024-11-21T11:16:46,997 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/9621f8b90d274258b815c1ef74e7fb0c, entries=150, sequenceid=213, filesize=11.9 K 2024-11-21T11:16:46,998 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 0e38bed8d085aac2e0e9aec5f988def4 in 624ms, sequenceid=213, compaction requested=true 2024-11-21T11:16:46,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 0e38bed8d085aac2e0e9aec5f988def4: 2024-11-21T11:16:46,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:46,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-21T11:16:46,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-21T11:16:47,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-21T11:16:47,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3200 sec 2024-11-21T11:16:47,003 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 2.3260 sec 2024-11-21T11:16:47,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:47,028 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0e38bed8d085aac2e0e9aec5f988def4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-21T11:16:47,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=A 2024-11-21T11:16:47,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:47,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=B 2024-11-21T11:16:47,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:47,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=C 2024-11-21T11:16:47,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:47,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112126f5da4b1ee64afeac16e7c5b65e8493_0e38bed8d085aac2e0e9aec5f988def4 is 50, key is test_row_0/A:col10/1732187806910/Put/seqid=0 2024-11-21T11:16:47,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742426_1602 (size=17284) 2024-11-21T11:16:47,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:47,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187867066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:47,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:47,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:47,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:47,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:47,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187867068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:47,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:47,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:47,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:47,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:47,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187867165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:47,167 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_1' in 0e38bed8d085aac2e0e9aec5f988def4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:16:47,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:47,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:47,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:47,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:47,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187867165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:47,170 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_1' in 0e38bed8d085aac2e0e9aec5f988def4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:16:47,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:47,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:47,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:47,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:47,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187867175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:47,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:47,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187867176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:47,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:47,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187867176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:47,179 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_2' in 0e38bed8d085aac2e0e9aec5f988def4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:16:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:47,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:47,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187867385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:47,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:47,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:47,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187867385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:47,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:47,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:47,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:47,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:47,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:47,440 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:47,443 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112126f5da4b1ee64afeac16e7c5b65e8493_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112126f5da4b1ee64afeac16e7c5b65e8493_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:47,444 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/9f9c39f79f48477081f64f3c3064f3b5, store: [table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:47,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/9f9c39f79f48477081f64f3c3064f3b5 is 175, key is test_row_0/A:col10/1732187806910/Put/seqid=0 2024-11-21T11:16:47,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742427_1603 (size=48389) 2024-11-21T11:16:47,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187867696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:47,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:47,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:47,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:47,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187867696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:47,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:47,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:47,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:47,848 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=227, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/9f9c39f79f48477081f64f3c3064f3b5 2024-11-21T11:16:47,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/afcc1df83a57439e89ba13ff5998daf3 is 50, key is test_row_0/B:col10/1732187806910/Put/seqid=0 2024-11-21T11:16:47,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742428_1604 (size=12151) 2024-11-21T11:16:48,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:48,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:48,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187868205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:48,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187868205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:48,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/afcc1df83a57439e89ba13ff5998daf3 2024-11-21T11:16:48,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/f14429c57a534f00a85f083d206bb2ee is 50, key is test_row_0/C:col10/1732187806910/Put/seqid=0 2024-11-21T11:16:48,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742429_1605 (size=12151) 2024-11-21T11:16:48,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/f14429c57a534f00a85f083d206bb2ee 2024-11-21T11:16:48,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/9f9c39f79f48477081f64f3c3064f3b5 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/9f9c39f79f48477081f64f3c3064f3b5 2024-11-21T11:16:48,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/9f9c39f79f48477081f64f3c3064f3b5, entries=250, sequenceid=227, filesize=47.3 K 2024-11-21T11:16:48,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/afcc1df83a57439e89ba13ff5998daf3 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/afcc1df83a57439e89ba13ff5998daf3 2024-11-21T11:16:48,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/afcc1df83a57439e89ba13ff5998daf3, entries=150, sequenceid=227, filesize=11.9 K 2024-11-21T11:16:48,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/f14429c57a534f00a85f083d206bb2ee as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/f14429c57a534f00a85f083d206bb2ee 2024-11-21T11:16:48,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/f14429c57a534f00a85f083d206bb2ee, entries=150, sequenceid=227, filesize=11.9 K 2024-11-21T11:16:48,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 0e38bed8d085aac2e0e9aec5f988def4 in 1656ms, sequenceid=227, compaction requested=true 2024-11-21T11:16:48,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0e38bed8d085aac2e0e9aec5f988def4: 2024-11-21T11:16:48,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e38bed8d085aac2e0e9aec5f988def4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:16:48,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:48,684 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T11:16:48,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e38bed8d085aac2e0e9aec5f988def4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:16:48,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:48,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e38bed8d085aac2e0e9aec5f988def4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:16:48,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:16:48,685 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T11:16:48,685 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150656 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T11:16:48,685 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T11:16:48,686 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 0e38bed8d085aac2e0e9aec5f988def4/A is initiating minor compaction (all files) 2024-11-21T11:16:48,686 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 0e38bed8d085aac2e0e9aec5f988def4/B is initiating minor compaction (all files) 2024-11-21T11:16:48,686 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e38bed8d085aac2e0e9aec5f988def4/A in TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:48,686 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e38bed8d085aac2e0e9aec5f988def4/B in TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:48,686 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/3b1654e6ae974160a5cad36a02c878bf, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/f3a43e878d904376b737e21c20abdd8f, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/ca28eac7eb8a4457897dd0f246cf1dfc, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/9f9c39f79f48477081f64f3c3064f3b5] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp, totalSize=147.1 K 2024-11-21T11:16:48,686 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/18a3599b08764dfb9c1d7253cb7dcf38, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/3109bb16e2274eeea247aa92f92a3c76, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/06e132731563462ba7116efeb25cdbfb, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/afcc1df83a57439e89ba13ff5998daf3] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp, totalSize=47.8 K 2024-11-21T11:16:48,686 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:48,686 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. files: [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/3b1654e6ae974160a5cad36a02c878bf, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/f3a43e878d904376b737e21c20abdd8f, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/ca28eac7eb8a4457897dd0f246cf1dfc, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/9f9c39f79f48477081f64f3c3064f3b5] 2024-11-21T11:16:48,686 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b1654e6ae974160a5cad36a02c878bf, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732187803769 2024-11-21T11:16:48,686 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 18a3599b08764dfb9c1d7253cb7dcf38, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732187803769 2024-11-21T11:16:48,686 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3a43e878d904376b737e21c20abdd8f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732187804692 2024-11-21T11:16:48,687 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 3109bb16e2274eeea247aa92f92a3c76, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732187804692 2024-11-21T11:16:48,687 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca28eac7eb8a4457897dd0f246cf1dfc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732187804996 2024-11-21T11:16:48,687 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 06e132731563462ba7116efeb25cdbfb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732187804996 2024-11-21T11:16:48,687 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f9c39f79f48477081f64f3c3064f3b5, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732187806896 2024-11-21T11:16:48,687 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting afcc1df83a57439e89ba13ff5998daf3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732187806910 2024-11-21T11:16:48,695 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e38bed8d085aac2e0e9aec5f988def4#B#compaction#509 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:48,695 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/ec6e63370be14ccab60e68139f92975a is 50, key is test_row_0/B:col10/1732187806910/Put/seqid=0 2024-11-21T11:16:48,696 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:48,701 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121b3e9b3e1de604ec4a33ad4feffa319c1_0e38bed8d085aac2e0e9aec5f988def4 store=[table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:48,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742430_1606 (size=12595) 2024-11-21T11:16:48,705 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121b3e9b3e1de604ec4a33ad4feffa319c1_0e38bed8d085aac2e0e9aec5f988def4, store=[table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:48,705 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b3e9b3e1de604ec4a33ad4feffa319c1_0e38bed8d085aac2e0e9aec5f988def4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:48,710 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/ec6e63370be14ccab60e68139f92975a as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/ec6e63370be14ccab60e68139f92975a 2024-11-21T11:16:48,715 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0e38bed8d085aac2e0e9aec5f988def4/B of 0e38bed8d085aac2e0e9aec5f988def4 into ec6e63370be14ccab60e68139f92975a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:48,715 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e38bed8d085aac2e0e9aec5f988def4: 2024-11-21T11:16:48,715 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., storeName=0e38bed8d085aac2e0e9aec5f988def4/B, priority=12, startTime=1732187808684; duration=0sec 2024-11-21T11:16:48,715 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:16:48,715 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e38bed8d085aac2e0e9aec5f988def4:B 2024-11-21T11:16:48,715 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T11:16:48,716 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T11:16:48,716 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 0e38bed8d085aac2e0e9aec5f988def4/C is initiating minor compaction (all files) 2024-11-21T11:16:48,716 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e38bed8d085aac2e0e9aec5f988def4/C in TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:48,717 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/3630d084e1394c09bc8c02841f65cab1, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/bdcc8883224448d78d2ee4798b2877e9, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/9621f8b90d274258b815c1ef74e7fb0c, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/f14429c57a534f00a85f083d206bb2ee] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp, totalSize=47.8 K 2024-11-21T11:16:48,717 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 3630d084e1394c09bc8c02841f65cab1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732187803769 2024-11-21T11:16:48,717 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting bdcc8883224448d78d2ee4798b2877e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732187804692 2024-11-21T11:16:48,717 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 9621f8b90d274258b815c1ef74e7fb0c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732187804996 2024-11-21T11:16:48,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742431_1607 (size=4469) 2024-11-21T11:16:48,718 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting f14429c57a534f00a85f083d206bb2ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732187806910 2024-11-21T11:16:48,722 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e38bed8d085aac2e0e9aec5f988def4#A#compaction#510 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:48,722 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/c5da8789822746ada72dd65286cf6d3d is 175, key is test_row_0/A:col10/1732187806910/Put/seqid=0 2024-11-21T11:16:48,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742432_1608 (size=31549) 2024-11-21T11:16:48,742 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/c5da8789822746ada72dd65286cf6d3d as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/c5da8789822746ada72dd65286cf6d3d 2024-11-21T11:16:48,747 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e38bed8d085aac2e0e9aec5f988def4#C#compaction#511 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:48,747 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0e38bed8d085aac2e0e9aec5f988def4/A of 0e38bed8d085aac2e0e9aec5f988def4 into c5da8789822746ada72dd65286cf6d3d(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:48,747 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e38bed8d085aac2e0e9aec5f988def4: 2024-11-21T11:16:48,747 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., storeName=0e38bed8d085aac2e0e9aec5f988def4/A, priority=12, startTime=1732187808684; duration=0sec 2024-11-21T11:16:48,747 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/a394d1a9c32f4548a244aa445c0d64a2 is 50, key is test_row_0/C:col10/1732187806910/Put/seqid=0 2024-11-21T11:16:48,747 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:48,747 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e38bed8d085aac2e0e9aec5f988def4:A 2024-11-21T11:16:48,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742433_1609 (size=12595) 2024-11-21T11:16:48,763 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/a394d1a9c32f4548a244aa445c0d64a2 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/a394d1a9c32f4548a244aa445c0d64a2 2024-11-21T11:16:48,768 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0e38bed8d085aac2e0e9aec5f988def4/C of 0e38bed8d085aac2e0e9aec5f988def4 into a394d1a9c32f4548a244aa445c0d64a2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:48,768 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e38bed8d085aac2e0e9aec5f988def4: 2024-11-21T11:16:48,768 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., storeName=0e38bed8d085aac2e0e9aec5f988def4/C, priority=12, startTime=1732187808685; duration=0sec 2024-11-21T11:16:48,768 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:48,768 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e38bed8d085aac2e0e9aec5f988def4:C 2024-11-21T11:16:48,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T11:16:48,846 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:16:48,846 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:16:48,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-21T11:16:48,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T11:16:48,848 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:16:48,849 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:16:48,849 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:16:48,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T11:16:49,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-21T11:16:49,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:49,001 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 0e38bed8d085aac2e0e9aec5f988def4 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-21T11:16:49,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=A 2024-11-21T11:16:49,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:49,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=B 2024-11-21T11:16:49,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:49,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=C 2024-11-21T11:16:49,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:49,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121480af24475fa432490efc3cf9522bb18_0e38bed8d085aac2e0e9aec5f988def4 is 50, key is test_row_0/A:col10/1732187807066/Put/seqid=0 2024-11-21T11:16:49,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742434_1610 (size=12304) 2024-11-21T11:16:49,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T11:16:49,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:49,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:49,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:49,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187869206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:49,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:49,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187869207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:49,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:49,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:49,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:49,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:49,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187869210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:49,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:49,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:49,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:49,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:49,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187869215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:49,216 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_0' in 0e38bed8d085aac2e0e9aec5f988def4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:16:49,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:49,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:49,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:49,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:49,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187869225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:49,229 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_0' in 0e38bed8d085aac2e0e9aec5f988def4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:16:49,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:49,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:49,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:49,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:49,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187869315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:49,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:49,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187869315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:49,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:49,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187869325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:49,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:49,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:49,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:49,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:49,416 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121480af24475fa432490efc3cf9522bb18_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121480af24475fa432490efc3cf9522bb18_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:49,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/a6eade662ee448cdbf763508fee73a16, store: [table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:49,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/a6eade662ee448cdbf763508fee73a16 is 175, key is test_row_0/A:col10/1732187807066/Put/seqid=0 2024-11-21T11:16:49,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742435_1611 (size=31105) 2024-11-21T11:16:49,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T11:16:49,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:49,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187869525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:49,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:49,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:49,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187869525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:49,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:49,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:49,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:49,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:49,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:49,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:49,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187869535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:49,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:49,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:49,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:49,821 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/a6eade662ee448cdbf763508fee73a16 2024-11-21T11:16:49,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/19752fb03714476493ad4414c669d00a is 50, key is test_row_0/B:col10/1732187807066/Put/seqid=0 2024-11-21T11:16:49,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742436_1612 (size=12151) 2024-11-21T11:16:49,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:49,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187869836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:49,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:49,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187869836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:49,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:49,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:49,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:49,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:49,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:49,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:49,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:49,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187869845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:49,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:49,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:49,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:49,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T11:16:50,235 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/19752fb03714476493ad4414c669d00a 2024-11-21T11:16:50,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/912029318c6043cc8126d86233c3fc4b is 50, key is test_row_0/C:col10/1732187807066/Put/seqid=0 2024-11-21T11:16:50,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742437_1613 (size=12151) 2024-11-21T11:16:50,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:50,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187870345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:50,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:50,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:50,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:50,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187870345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:50,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:50,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:50,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:50,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:50,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:50,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187870356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:50,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:50,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:50,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:50,645 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/912029318c6043cc8126d86233c3fc4b 2024-11-21T11:16:50,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/a6eade662ee448cdbf763508fee73a16 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/a6eade662ee448cdbf763508fee73a16 2024-11-21T11:16:50,652 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/a6eade662ee448cdbf763508fee73a16, entries=150, sequenceid=248, filesize=30.4 K 2024-11-21T11:16:50,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/19752fb03714476493ad4414c669d00a as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/19752fb03714476493ad4414c669d00a 2024-11-21T11:16:50,656 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/19752fb03714476493ad4414c669d00a, entries=150, sequenceid=248, filesize=11.9 K 2024-11-21T11:16:50,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/912029318c6043cc8126d86233c3fc4b as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/912029318c6043cc8126d86233c3fc4b 2024-11-21T11:16:50,660 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/912029318c6043cc8126d86233c3fc4b, entries=150, sequenceid=248, filesize=11.9 K 2024-11-21T11:16:50,661 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 0e38bed8d085aac2e0e9aec5f988def4 in 1660ms, sequenceid=248, compaction requested=false 2024-11-21T11:16:50,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 0e38bed8d085aac2e0e9aec5f988def4: 2024-11-21T11:16:50,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:50,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-21T11:16:50,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-21T11:16:50,665 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-21T11:16:50,665 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8140 sec 2024-11-21T11:16:50,668 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.8200 sec 2024-11-21T11:16:51,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T11:16:51,005 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:16:51,006 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:16:51,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-21T11:16:51,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-21T11:16:51,008 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:16:51,008 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:16:51,008 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:16:51,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-21T11:16:51,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-21T11:16:51,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:51,160 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 0e38bed8d085aac2e0e9aec5f988def4 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T11:16:51,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=A 2024-11-21T11:16:51,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:51,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=B 2024-11-21T11:16:51,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:51,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=C 2024-11-21T11:16:51,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:51,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c9a45584609c44b0b631858a20f1c95b_0e38bed8d085aac2e0e9aec5f988def4 is 50, key is test_row_0/A:col10/1732187809206/Put/seqid=0 2024-11-21T11:16:51,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742438_1614 (size=12454) 2024-11-21T11:16:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:51,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:51,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:51,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187871273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:51,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:51,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:51,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:51,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:51,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187871279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:51,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:51,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:51,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-21T11:16:51,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:51,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187871355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:51,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187871355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:51,359 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_2' in 0e38bed8d085aac2e0e9aec5f988def4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:16:51,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:51,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:51,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:51,359 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_1' in 0e38bed8d085aac2e0e9aec5f988def4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:16:51,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:51,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:51,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:51,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:51,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187871365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:51,369 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 7b462513bfc2,33417,1732187618276 for 'test_row_1' in 0e38bed8d085aac2e0e9aec5f988def4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T11:16:51,369 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:51,369 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:51,369 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:51,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187871385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:51,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:51,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:51,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:51,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187871395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:51,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:51,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:51,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:51,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:51,573 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c9a45584609c44b0b631858a20f1c95b_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c9a45584609c44b0b631858a20f1c95b_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:51,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/dbbe5d7ea6464363bbeeae29eb0b3a5f, store: [table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:51,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/dbbe5d7ea6464363bbeeae29eb0b3a5f is 175, key is test_row_0/A:col10/1732187809206/Put/seqid=0 2024-11-21T11:16:51,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742439_1615 (size=31255) 2024-11-21T11:16:51,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187871595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:51,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187871606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:51,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:51,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:51,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:51,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-21T11:16:51,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187871905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:51,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:51,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:51,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:51,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:51,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187871916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:51,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:51,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:51,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:51,979 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=262, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/dbbe5d7ea6464363bbeeae29eb0b3a5f 2024-11-21T11:16:51,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/9b568837ae2f40c0aa5d0faf3e70a772 is 50, key is test_row_0/B:col10/1732187809206/Put/seqid=0 2024-11-21T11:16:51,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742440_1616 (size=12301) 2024-11-21T11:16:51,989 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/9b568837ae2f40c0aa5d0faf3e70a772 2024-11-21T11:16:51,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/67bb2eea54e34aac8ad4e3754726077d is 50, key is test_row_0/C:col10/1732187809206/Put/seqid=0 2024-11-21T11:16:51,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742441_1617 (size=12301) 2024-11-21T11:16:51,999 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/67bb2eea54e34aac8ad4e3754726077d 2024-11-21T11:16:52,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/dbbe5d7ea6464363bbeeae29eb0b3a5f as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/dbbe5d7ea6464363bbeeae29eb0b3a5f 2024-11-21T11:16:52,006 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/dbbe5d7ea6464363bbeeae29eb0b3a5f, entries=150, sequenceid=262, filesize=30.5 K 2024-11-21T11:16:52,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/9b568837ae2f40c0aa5d0faf3e70a772 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/9b568837ae2f40c0aa5d0faf3e70a772 2024-11-21T11:16:52,011 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/9b568837ae2f40c0aa5d0faf3e70a772, entries=150, sequenceid=262, filesize=12.0 K 2024-11-21T11:16:52,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/67bb2eea54e34aac8ad4e3754726077d as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/67bb2eea54e34aac8ad4e3754726077d 2024-11-21T11:16:52,014 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/67bb2eea54e34aac8ad4e3754726077d, entries=150, sequenceid=262, filesize=12.0 K 2024-11-21T11:16:52,015 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 0e38bed8d085aac2e0e9aec5f988def4 in 855ms, sequenceid=262, compaction requested=true 2024-11-21T11:16:52,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 0e38bed8d085aac2e0e9aec5f988def4: 2024-11-21T11:16:52,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:52,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-21T11:16:52,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-21T11:16:52,018 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-21T11:16:52,018 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0090 sec 2024-11-21T11:16:52,020 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 1.0130 sec 2024-11-21T11:16:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-21T11:16:52,146 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:16:52,146 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:16:52,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-21T11:16:52,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-21T11:16:52,148 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:16:52,148 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:16:52,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:16:52,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-21T11:16:52,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-21T11:16:52,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:52,301 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing 0e38bed8d085aac2e0e9aec5f988def4 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-21T11:16:52,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=A 2024-11-21T11:16:52,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:52,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=B 2024-11-21T11:16:52,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:52,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=C 2024-11-21T11:16:52,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:52,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ced705b61c67477b941d4225c5ac10e5_0e38bed8d085aac2e0e9aec5f988def4 is 50, key is test_row_0/A:col10/1732187811247/Put/seqid=0 2024-11-21T11:16:52,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742442_1618 (size=12454) 2024-11-21T11:16:52,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:52,315 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ced705b61c67477b941d4225c5ac10e5_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ced705b61c67477b941d4225c5ac10e5_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:52,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/a1e7988013334eabaee2aff90aebcbd3, store: [table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:52,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/a1e7988013334eabaee2aff90aebcbd3 is 175, key is test_row_0/A:col10/1732187811247/Put/seqid=0 2024-11-21T11:16:52,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742443_1619 (size=31255) 2024-11-21T11:16:52,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:52,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:52,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:52,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187872444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:52,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:52,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:52,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:52,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187872448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:52,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:52,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:52,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-21T11:16:52,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187872555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:52,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:52,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:52,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:52,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187872555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:52,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:52,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:52,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:52,722 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=280, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/a1e7988013334eabaee2aff90aebcbd3 2024-11-21T11:16:52,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/bb2e0d1ac40f4959b554738086324ed8 is 50, key is test_row_0/B:col10/1732187811247/Put/seqid=0 2024-11-21T11:16:52,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742444_1620 (size=12301) 2024-11-21T11:16:52,732 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/bb2e0d1ac40f4959b554738086324ed8 2024-11-21T11:16:52,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/b3dadeffa489450388a5b724307a61ae is 50, key is test_row_0/C:col10/1732187811247/Put/seqid=0 2024-11-21T11:16:52,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742445_1621 (size=12301) 2024-11-21T11:16:52,746 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/b3dadeffa489450388a5b724307a61ae 2024-11-21T11:16:52,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/a1e7988013334eabaee2aff90aebcbd3 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/a1e7988013334eabaee2aff90aebcbd3 2024-11-21T11:16:52,753 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/a1e7988013334eabaee2aff90aebcbd3, entries=150, sequenceid=280, filesize=30.5 K 2024-11-21T11:16:52,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/bb2e0d1ac40f4959b554738086324ed8 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/bb2e0d1ac40f4959b554738086324ed8 2024-11-21T11:16:52,758 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/bb2e0d1ac40f4959b554738086324ed8, entries=150, sequenceid=280, filesize=12.0 K 2024-11-21T11:16:52,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/b3dadeffa489450388a5b724307a61ae as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/b3dadeffa489450388a5b724307a61ae 2024-11-21T11:16:52,764 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/b3dadeffa489450388a5b724307a61ae, entries=150, sequenceid=280, filesize=12.0 K 2024-11-21T11:16:52,764 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 0e38bed8d085aac2e0e9aec5f988def4 in 463ms, sequenceid=280, compaction requested=true 2024-11-21T11:16:52,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for 0e38bed8d085aac2e0e9aec5f988def4: 2024-11-21T11:16:52,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:52,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-21T11:16:52,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-21T11:16:52,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:52,768 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0e38bed8d085aac2e0e9aec5f988def4 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-21T11:16:52,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=A 2024-11-21T11:16:52,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:52,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=B 2024-11-21T11:16:52,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:52,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=C 2024-11-21T11:16:52,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:52,771 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-21T11:16:52,771 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 618 msec 2024-11-21T11:16:52,773 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 625 msec 2024-11-21T11:16:52,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-21T11:16:52,775 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:16:52,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215cad77b667934e5cb468097b609b78c4_0e38bed8d085aac2e0e9aec5f988def4 is 50, key is test_row_0/A:col10/1732187812766/Put/seqid=0 2024-11-21T11:16:52,776 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:16:52,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees 2024-11-21T11:16:52,778 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:16:52,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-21T11:16:52,778 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:16:52,778 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:16:52,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742446_1622 (size=12454) 2024-11-21T11:16:52,782 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:52,785 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215cad77b667934e5cb468097b609b78c4_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215cad77b667934e5cb468097b609b78c4_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:52,786 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/5d0a3c2b7e2d40edbb0be29950cf229e, store: [table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:52,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/5d0a3c2b7e2d40edbb0be29950cf229e is 175, key is test_row_0/A:col10/1732187812766/Put/seqid=0 2024-11-21T11:16:52,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742447_1623 (size=31255) 2024-11-21T11:16:52,789 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=295, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/5d0a3c2b7e2d40edbb0be29950cf229e 2024-11-21T11:16:52,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/419fb85eace14fd087a87a0d802c6273 is 50, key is test_row_0/B:col10/1732187812766/Put/seqid=0 2024-11-21T11:16:52,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742448_1624 (size=12301) 2024-11-21T11:16:52,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:52,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187872813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:52,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:52,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:52,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:52,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:52,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187872814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:52,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:52,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:52,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:52,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-21T11:16:52,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:52,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187872925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:52,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:52,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187872926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:52,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:52,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:52,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:52,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:52,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:52,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:52,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-21T11:16:52,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:52,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:52,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:52,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:52,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:52,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:53,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-21T11:16:53,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:53,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:53,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:53,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:53,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:53,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:53,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-21T11:16:53,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:53,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187873135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:53,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:53,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:53,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187873135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:53,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:53,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:53,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:53,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:53,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:53,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/419fb85eace14fd087a87a0d802c6273 2024-11-21T11:16:53,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/3b116d9990d0488dbd6b0429ba7bc4cb is 50, key is test_row_0/C:col10/1732187812766/Put/seqid=0 2024-11-21T11:16:53,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742449_1625 (size=12301) 2024-11-21T11:16:53,212 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/3b116d9990d0488dbd6b0429ba7bc4cb 2024-11-21T11:16:53,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/5d0a3c2b7e2d40edbb0be29950cf229e as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/5d0a3c2b7e2d40edbb0be29950cf229e 2024-11-21T11:16:53,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/5d0a3c2b7e2d40edbb0be29950cf229e, entries=150, sequenceid=295, filesize=30.5 K 2024-11-21T11:16:53,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/419fb85eace14fd087a87a0d802c6273 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/419fb85eace14fd087a87a0d802c6273 2024-11-21T11:16:53,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/419fb85eace14fd087a87a0d802c6273, entries=150, sequenceid=295, filesize=12.0 K 2024-11-21T11:16:53,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/3b116d9990d0488dbd6b0429ba7bc4cb as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/3b116d9990d0488dbd6b0429ba7bc4cb 2024-11-21T11:16:53,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/3b116d9990d0488dbd6b0429ba7bc4cb, entries=150, sequenceid=295, filesize=12.0 K 2024-11-21T11:16:53,230 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 0e38bed8d085aac2e0e9aec5f988def4 in 462ms, sequenceid=295, compaction requested=true 2024-11-21T11:16:53,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0e38bed8d085aac2e0e9aec5f988def4: 2024-11-21T11:16:53,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e38bed8d085aac2e0e9aec5f988def4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:16:53,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:53,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e38bed8d085aac2e0e9aec5f988def4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:16:53,231 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-21T11:16:53,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:16:53,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e38bed8d085aac2e0e9aec5f988def4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:16:53,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T11:16:53,231 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-21T11:16:53,232 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61649 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-21T11:16:53,232 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 156419 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-21T11:16:53,232 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 0e38bed8d085aac2e0e9aec5f988def4/B is initiating minor compaction (all files) 2024-11-21T11:16:53,232 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 0e38bed8d085aac2e0e9aec5f988def4/A is initiating minor compaction (all files) 2024-11-21T11:16:53,232 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e38bed8d085aac2e0e9aec5f988def4/A in TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:53,232 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e38bed8d085aac2e0e9aec5f988def4/B in TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:53,232 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/c5da8789822746ada72dd65286cf6d3d, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/a6eade662ee448cdbf763508fee73a16, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/dbbe5d7ea6464363bbeeae29eb0b3a5f, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/a1e7988013334eabaee2aff90aebcbd3, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/5d0a3c2b7e2d40edbb0be29950cf229e] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp, totalSize=152.8 K 2024-11-21T11:16:53,233 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:53,233 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/ec6e63370be14ccab60e68139f92975a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/19752fb03714476493ad4414c669d00a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/9b568837ae2f40c0aa5d0faf3e70a772, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/bb2e0d1ac40f4959b554738086324ed8, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/419fb85eace14fd087a87a0d802c6273] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp, totalSize=60.2 K 2024-11-21T11:16:53,233 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. files: [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/c5da8789822746ada72dd65286cf6d3d, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/a6eade662ee448cdbf763508fee73a16, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/dbbe5d7ea6464363bbeeae29eb0b3a5f, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/a1e7988013334eabaee2aff90aebcbd3, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/5d0a3c2b7e2d40edbb0be29950cf229e] 2024-11-21T11:16:53,233 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec6e63370be14ccab60e68139f92975a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732187806910 2024-11-21T11:16:53,233 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting c5da8789822746ada72dd65286cf6d3d, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732187806910 2024-11-21T11:16:53,233 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting a6eade662ee448cdbf763508fee73a16, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732187807035 2024-11-21T11:16:53,233 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19752fb03714476493ad4414c669d00a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732187807035 2024-11-21T11:16:53,233 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b568837ae2f40c0aa5d0faf3e70a772, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732187809205 2024-11-21T11:16:53,233 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting dbbe5d7ea6464363bbeeae29eb0b3a5f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732187809205 2024-11-21T11:16:53,234 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb2e0d1ac40f4959b554738086324ed8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732187811247 2024-11-21T11:16:53,234 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting a1e7988013334eabaee2aff90aebcbd3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732187811247 2024-11-21T11:16:53,234 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 419fb85eace14fd087a87a0d802c6273, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732187812442 2024-11-21T11:16:53,234 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d0a3c2b7e2d40edbb0be29950cf229e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732187812442 2024-11-21T11:16:53,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-21T11:16:53,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:53,237 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2902): Flushing 0e38bed8d085aac2e0e9aec5f988def4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-21T11:16:53,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=A 2024-11-21T11:16:53,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:53,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=B 2024-11-21T11:16:53,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:53,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=C 2024-11-21T11:16:53,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:53,244 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:53,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c4085de37cc44d0cabc8092ee31c615d_0e38bed8d085aac2e0e9aec5f988def4 is 50, key is test_row_0/A:col10/1732187812804/Put/seqid=0 2024-11-21T11:16:53,247 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e38bed8d085aac2e0e9aec5f988def4#B#compaction#524 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:53,247 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121ea4f60df14cf4e16a91ab0f38baf415c_0e38bed8d085aac2e0e9aec5f988def4 store=[table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:53,247 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/2e904ba4307f41329717f6d990da3471 is 50, key is test_row_0/B:col10/1732187812766/Put/seqid=0 2024-11-21T11:16:53,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742450_1626 (size=12454) 2024-11-21T11:16:53,250 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121ea4f60df14cf4e16a91ab0f38baf415c_0e38bed8d085aac2e0e9aec5f988def4, store=[table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:53,250 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ea4f60df14cf4e16a91ab0f38baf415c_0e38bed8d085aac2e0e9aec5f988def4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:53,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,254 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c4085de37cc44d0cabc8092ee31c615d_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c4085de37cc44d0cabc8092ee31c615d_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:53,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/cf4d19e87c5c44ccbc86be892ae93fae, store: [table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:53,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/cf4d19e87c5c44ccbc86be892ae93fae is 175, key is test_row_0/A:col10/1732187812804/Put/seqid=0 2024-11-21T11:16:53,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742451_1627 (size=12915) 2024-11-21T11:16:53,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742452_1628 (size=4469) 2024-11-21T11:16:53,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742453_1629 (size=31255) 2024-11-21T11:16:53,261 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/cf4d19e87c5c44ccbc86be892ae93fae 2024-11-21T11:16:53,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/291696e65c734417bd4c3abd325d9905 is 50, key is test_row_0/B:col10/1732187812804/Put/seqid=0 2024-11-21T11:16:53,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742454_1630 (size=12301) 2024-11-21T11:16:53,275 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/291696e65c734417bd4c3abd325d9905 2024-11-21T11:16:53,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/88db3af2e05644c892d1265fdc6f81db is 50, key is test_row_0/C:col10/1732187812804/Put/seqid=0 2024-11-21T11:16:53,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742455_1631 (size=12301) 2024-11-21T11:16:53,289 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/88db3af2e05644c892d1265fdc6f81db 2024-11-21T11:16:53,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/cf4d19e87c5c44ccbc86be892ae93fae as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/cf4d19e87c5c44ccbc86be892ae93fae 2024-11-21T11:16:53,297 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/cf4d19e87c5c44ccbc86be892ae93fae, entries=150, sequenceid=312, filesize=30.5 K 2024-11-21T11:16:53,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/291696e65c734417bd4c3abd325d9905 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/291696e65c734417bd4c3abd325d9905 2024-11-21T11:16:53,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,301 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/291696e65c734417bd4c3abd325d9905, entries=150, sequenceid=312, filesize=12.0 K 2024-11-21T11:16:53,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/88db3af2e05644c892d1265fdc6f81db as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/88db3af2e05644c892d1265fdc6f81db 2024-11-21T11:16:53,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,305 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/88db3af2e05644c892d1265fdc6f81db, entries=150, sequenceid=312, filesize=12.0 K 2024-11-21T11:16:53,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,305 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for 0e38bed8d085aac2e0e9aec5f988def4 in 68ms, sequenceid=312, compaction requested=true 2024-11-21T11:16:53,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2603): Flush status journal for 0e38bed8d085aac2e0e9aec5f988def4: 2024-11-21T11:16:53,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:53,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=173 2024-11-21T11:16:53,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=173 2024-11-21T11:16:53,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,308 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-21T11:16:53,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,309 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 529 msec 2024-11-21T11:16:53,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,310 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees in 533 msec 2024-11-21T11:16:53,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-21T11:16:53,405 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:16:53,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,406 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T11:16:53,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees 2024-11-21T11:16:53,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-21T11:16:53,408 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T11:16:53,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,408 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T11:16:53,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,408 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T11:16:53,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:53,423 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0e38bed8d085aac2e0e9aec5f988def4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T11:16:53,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=A 2024-11-21T11:16:53,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:53,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=B 2024-11-21T11:16:53,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:53,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=C 2024-11-21T11:16:53,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:53,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112199779836ec3f40e4b5522ede9e8f04d8_0e38bed8d085aac2e0e9aec5f988def4 is 50, key is test_row_0/A:col10/1732187813421/Put/seqid=0 2024-11-21T11:16:53,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742456_1632 (size=14994) 2024-11-21T11:16:53,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732187813450, max timestamp=1732187813458Num uniques 859; 2024-11-21T11:16:53,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T11:16:53,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732187813450, max timestamp=1732187813458Num uniques 859; 2024-11-21T11:16:53,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7b462513bfc2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T11:16:53,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:53,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187873472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:53,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:53,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:53,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:53,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187873473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:53,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:53,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:53,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:53,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187873473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:53,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:53,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:53,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:53,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:53,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:53,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187873474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:53,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:53,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:53,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:53,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:53,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:53,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187873475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:53,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:53,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:53,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:53,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-21T11:16:53,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-21T11:16:53,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:53,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:53,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:53,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:53,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:53,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:53,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:53,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187873585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:53,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:53,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:53,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187873585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:53,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:53,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:53,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:53,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187873585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:53,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:53,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:53,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187873586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:53,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:53,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187873586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:53,658 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e38bed8d085aac2e0e9aec5f988def4#A#compaction#526 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:53,659 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/01bd9a11edb04839b72265e3823b6923 is 175, key is test_row_0/A:col10/1732187812766/Put/seqid=0 2024-11-21T11:16:53,662 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/2e904ba4307f41329717f6d990da3471 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/2e904ba4307f41329717f6d990da3471 2024-11-21T11:16:53,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742457_1633 (size=31869) 2024-11-21T11:16:53,666 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 0e38bed8d085aac2e0e9aec5f988def4/B of 0e38bed8d085aac2e0e9aec5f988def4 into 2e904ba4307f41329717f6d990da3471(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:53,667 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e38bed8d085aac2e0e9aec5f988def4: 2024-11-21T11:16:53,667 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., storeName=0e38bed8d085aac2e0e9aec5f988def4/B, priority=11, startTime=1732187813231; duration=0sec 2024-11-21T11:16:53,667 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:16:53,667 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e38bed8d085aac2e0e9aec5f988def4:B 2024-11-21T11:16:53,667 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-21T11:16:53,668 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73950 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-21T11:16:53,668 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 0e38bed8d085aac2e0e9aec5f988def4/C is initiating minor compaction (all files) 2024-11-21T11:16:53,668 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e38bed8d085aac2e0e9aec5f988def4/C in TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:53,668 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/a394d1a9c32f4548a244aa445c0d64a2, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/912029318c6043cc8126d86233c3fc4b, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/67bb2eea54e34aac8ad4e3754726077d, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/b3dadeffa489450388a5b724307a61ae, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/3b116d9990d0488dbd6b0429ba7bc4cb, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/88db3af2e05644c892d1265fdc6f81db] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp, totalSize=72.2 K 2024-11-21T11:16:53,669 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting a394d1a9c32f4548a244aa445c0d64a2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732187806910 2024-11-21T11:16:53,669 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 912029318c6043cc8126d86233c3fc4b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732187807035 2024-11-21T11:16:53,670 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67bb2eea54e34aac8ad4e3754726077d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732187809205 2024-11-21T11:16:53,670 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3dadeffa489450388a5b724307a61ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732187811247 2024-11-21T11:16:53,670 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b116d9990d0488dbd6b0429ba7bc4cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732187812442 2024-11-21T11:16:53,670 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88db3af2e05644c892d1265fdc6f81db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732187812804 2024-11-21T11:16:53,679 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e38bed8d085aac2e0e9aec5f988def4#C#compaction#530 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:53,679 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/b6b9d74d2add460f8a90aa7ec79862be is 50, key is test_row_0/C:col10/1732187812804/Put/seqid=0 2024-11-21T11:16:53,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742458_1634 (size=12949) 2024-11-21T11:16:53,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-21T11:16:53,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:53,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:53,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:53,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:53,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:53,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:53,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-21T11:16:53,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:53,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187873795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:53,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:53,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187873796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:53,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:53,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:53,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:53,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:53,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:53,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187873796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:53,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:53,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:53,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:53,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187873796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:53,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:53,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:53,801 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187873796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:53,801 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:53,801 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:53,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:53,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:53,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:53,834 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:53,837 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112199779836ec3f40e4b5522ede9e8f04d8_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112199779836ec3f40e4b5522ede9e8f04d8_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:53,838 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/40eb5437dfa141cc873706bb9c910823, store: [table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:53,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/40eb5437dfa141cc873706bb9c910823 is 175, key is test_row_0/A:col10/1732187813421/Put/seqid=0 2024-11-21T11:16:53,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742459_1635 (size=39949) 2024-11-21T11:16:53,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-21T11:16:53,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:53,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:53,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:53,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:53,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:53,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:53,929 INFO [Thread-2489 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2489. 2024-11-21T11:16:53,929 DEBUG [Thread-2489 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:16:53,929 DEBUG [Thread-2489 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:53,929 DEBUG [Thread-2489 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:53,929 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:16:53,930 INFO [Thread-2481 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2481. 2024-11-21T11:16:53,930 DEBUG [Thread-2481 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:16:53,930 DEBUG [Thread-2481 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:53,931 DEBUG [Thread-2481 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:53,931 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:16:53,934 INFO [Thread-2483 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2483. 2024-11-21T11:16:53,934 DEBUG [Thread-2483 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:16:53,934 DEBUG [Thread-2483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:53,934 DEBUG [Thread-2483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:53,934 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:16:53,935 INFO [Thread-2487 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2487. 2024-11-21T11:16:53,935 DEBUG [Thread-2487 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:16:53,935 DEBUG [Thread-2487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:53,935 DEBUG [Thread-2487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:53,935 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:16:53,936 INFO [Thread-2485 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2485. 2024-11-21T11:16:53,936 DEBUG [Thread-2485 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:16:53,936 DEBUG [Thread-2485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:53,937 DEBUG [Thread-2485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:53,937 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:16:54,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-21T11:16:54,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:54,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:54,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:54,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:54,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:54,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:54,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-21T11:16:54,067 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/01bd9a11edb04839b72265e3823b6923 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/01bd9a11edb04839b72265e3823b6923 2024-11-21T11:16:54,070 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 0e38bed8d085aac2e0e9aec5f988def4/A of 0e38bed8d085aac2e0e9aec5f988def4 into 01bd9a11edb04839b72265e3823b6923(size=31.1 K), total size for store is 61.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:54,070 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e38bed8d085aac2e0e9aec5f988def4: 2024-11-21T11:16:54,070 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., storeName=0e38bed8d085aac2e0e9aec5f988def4/A, priority=11, startTime=1732187813230; duration=0sec 2024-11-21T11:16:54,070 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:54,070 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e38bed8d085aac2e0e9aec5f988def4:A 2024-11-21T11:16:54,085 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/b6b9d74d2add460f8a90aa7ec79862be as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/b6b9d74d2add460f8a90aa7ec79862be 2024-11-21T11:16:54,088 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 0e38bed8d085aac2e0e9aec5f988def4/C of 0e38bed8d085aac2e0e9aec5f988def4 into b6b9d74d2add460f8a90aa7ec79862be(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:54,088 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e38bed8d085aac2e0e9aec5f988def4: 2024-11-21T11:16:54,088 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., storeName=0e38bed8d085aac2e0e9aec5f988def4/C, priority=10, startTime=1732187813231; duration=0sec 2024-11-21T11:16:54,088 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:54,088 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e38bed8d085aac2e0e9aec5f988def4:C 2024-11-21T11:16:54,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:54,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:54,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:54,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187874105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:54,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187874105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:54,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187874105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:54,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:54,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:54,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187874106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:54,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:54,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:54,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187874106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:54,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:54,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:54,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:54,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:54,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:54,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:54,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:54,106 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:54,106 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:54,106 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:54,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:54,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:54,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:54,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-21T11:16:54,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:54,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:54,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:54,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:54,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:54,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:54,243 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=322, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/40eb5437dfa141cc873706bb9c910823 2024-11-21T11:16:54,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/d423cb5c77314ed1991f2345c5c274a4 is 50, key is test_row_0/B:col10/1732187813421/Put/seqid=0 2024-11-21T11:16:54,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742460_1636 (size=12301) 2024-11-21T11:16:54,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-21T11:16:54,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:54,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:54,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:54,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:54,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:54,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-21T11:16:54,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:54,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:54,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:54,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:54,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:54,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-21T11:16:54,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:54,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:54,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732187874615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34278 deadline: 1732187874615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34274 deadline: 1732187874615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:54,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34344 deadline: 1732187874616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:54,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:54,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:54,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:54,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:16:54,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:54,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:54,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:54,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:54,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:54,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34294 deadline: 1732187874616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:54,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:54,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:54,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:54,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 , the old value is region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:54,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T11:16:54,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., hostname=7b462513bfc2,33417,1732187618276, seqNum=5 because the exception is null or not the one we care about 2024-11-21T11:16:54,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-21T11:16:54,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:54,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:54,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:54,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:54,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:54,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:54,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/d423cb5c77314ed1991f2345c5c274a4 2024-11-21T11:16:54,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/e29f3c107d9b4295b28b9419920421b0 is 50, key is test_row_0/C:col10/1732187813421/Put/seqid=0 2024-11-21T11:16:54,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742461_1637 (size=12301) 2024-11-21T11:16:54,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-21T11:16:54,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:54,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:54,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:54,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:54,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:54,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-21T11:16:54,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:54,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:54,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:54,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:54,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:16:55,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/e29f3c107d9b4295b28b9419920421b0 2024-11-21T11:16:55,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/40eb5437dfa141cc873706bb9c910823 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/40eb5437dfa141cc873706bb9c910823 2024-11-21T11:16:55,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/40eb5437dfa141cc873706bb9c910823, entries=200, sequenceid=322, filesize=39.0 K 2024-11-21T11:16:55,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/d423cb5c77314ed1991f2345c5c274a4 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/d423cb5c77314ed1991f2345c5c274a4 2024-11-21T11:16:55,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/d423cb5c77314ed1991f2345c5c274a4, entries=150, sequenceid=322, filesize=12.0 K 2024-11-21T11:16:55,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/e29f3c107d9b4295b28b9419920421b0 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/e29f3c107d9b4295b28b9419920421b0 2024-11-21T11:16:55,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/e29f3c107d9b4295b28b9419920421b0, entries=150, sequenceid=322, filesize=12.0 K 2024-11-21T11:16:55,073 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=129.71 KB/132820 for 0e38bed8d085aac2e0e9aec5f988def4 in 1650ms, sequenceid=322, compaction requested=true 2024-11-21T11:16:55,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0e38bed8d085aac2e0e9aec5f988def4: 2024-11-21T11:16:55,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e38bed8d085aac2e0e9aec5f988def4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T11:16:55,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:55,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e38bed8d085aac2e0e9aec5f988def4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T11:16:55,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:55,073 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:16:55,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e38bed8d085aac2e0e9aec5f988def4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T11:16:55,073 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T11:16:55,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:16:55,074 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:16:55,074 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T11:16:55,074 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 0e38bed8d085aac2e0e9aec5f988def4/A is initiating minor compaction (all files) 2024-11-21T11:16:55,074 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1541): 0e38bed8d085aac2e0e9aec5f988def4/B is initiating minor compaction (all files) 2024-11-21T11:16:55,074 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e38bed8d085aac2e0e9aec5f988def4/A in TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:55,074 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e38bed8d085aac2e0e9aec5f988def4/B in TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:55,074 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/01bd9a11edb04839b72265e3823b6923, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/cf4d19e87c5c44ccbc86be892ae93fae, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/40eb5437dfa141cc873706bb9c910823] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp, totalSize=100.7 K 2024-11-21T11:16:55,074 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:55,074 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/2e904ba4307f41329717f6d990da3471, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/291696e65c734417bd4c3abd325d9905, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/d423cb5c77314ed1991f2345c5c274a4] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp, totalSize=36.6 K 2024-11-21T11:16:55,074 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. files: [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/01bd9a11edb04839b72265e3823b6923, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/cf4d19e87c5c44ccbc86be892ae93fae, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/40eb5437dfa141cc873706bb9c910823] 2024-11-21T11:16:55,074 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e904ba4307f41329717f6d990da3471, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732187812442 2024-11-21T11:16:55,074 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01bd9a11edb04839b72265e3823b6923, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732187812442 2024-11-21T11:16:55,074 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting 291696e65c734417bd4c3abd325d9905, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732187812804 2024-11-21T11:16:55,074 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf4d19e87c5c44ccbc86be892ae93fae, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732187812804 2024-11-21T11:16:55,075 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.Compactor(225): Compacting d423cb5c77314ed1991f2345c5c274a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732187813410 2024-11-21T11:16:55,075 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40eb5437dfa141cc873706bb9c910823, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732187813410 2024-11-21T11:16:55,080 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:55,080 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e38bed8d085aac2e0e9aec5f988def4#B#compaction#533 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:55,081 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112151971ec4181340fe99d0c2634e27dfbe_0e38bed8d085aac2e0e9aec5f988def4 store=[table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:55,081 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/5026cb27a0f440c2ad24e62ad1a40258 is 50, key is test_row_0/B:col10/1732187813421/Put/seqid=0 2024-11-21T11:16:55,084 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112151971ec4181340fe99d0c2634e27dfbe_0e38bed8d085aac2e0e9aec5f988def4, store=[table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:55,084 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112151971ec4181340fe99d0c2634e27dfbe_0e38bed8d085aac2e0e9aec5f988def4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:55,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-21T11:16:55,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:55,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742462_1638 (size=13017) 2024-11-21T11:16:55,088 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2902): Flushing 0e38bed8d085aac2e0e9aec5f988def4 3/3 column families, dataSize=129.71 KB heapSize=373.55 KB 2024-11-21T11:16:55,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=A 2024-11-21T11:16:55,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:16:55,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=B 2024-11-21T11:16:55,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:16:55,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=C 2024-11-21T11:16:55,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T11:16:55,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742463_1639 (size=4469) 2024-11-21T11:16:55,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c2754dc2d189424b9b030916e03f119e_0e38bed8d085aac2e0e9aec5f988def4 is 50, key is test_row_0/A:col10/1732187813471/Put/seqid=0 2024-11-21T11:16:55,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742464_1640 (size=12454) 2024-11-21T11:16:55,491 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e38bed8d085aac2e0e9aec5f988def4#A#compaction#534 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T11:16:55,491 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/5026cb27a0f440c2ad24e62ad1a40258 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/5026cb27a0f440c2ad24e62ad1a40258 2024-11-21T11:16:55,491 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/9334b487a2be42c99c4addf8afece55c is 175, key is test_row_0/A:col10/1732187813421/Put/seqid=0 2024-11-21T11:16:55,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742465_1641 (size=31971) 2024-11-21T11:16:55,495 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0e38bed8d085aac2e0e9aec5f988def4/B of 0e38bed8d085aac2e0e9aec5f988def4 into 5026cb27a0f440c2ad24e62ad1a40258(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:55,495 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e38bed8d085aac2e0e9aec5f988def4: 2024-11-21T11:16:55,495 INFO [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., storeName=0e38bed8d085aac2e0e9aec5f988def4/B, priority=13, startTime=1732187815073; duration=0sec 2024-11-21T11:16:55,495 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T11:16:55,495 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e38bed8d085aac2e0e9aec5f988def4:B 2024-11-21T11:16:55,495 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-21T11:16:55,496 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-21T11:16:55,496 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-21T11:16:55,496 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. because compaction request was cancelled 2024-11-21T11:16:55,496 DEBUG [RS:0;7b462513bfc2:33417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e38bed8d085aac2e0e9aec5f988def4:C 2024-11-21T11:16:55,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:55,499 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c2754dc2d189424b9b030916e03f119e_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c2754dc2d189424b9b030916e03f119e_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:55,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/94b6b4e762e742349513aff50fb642d7, store: [table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:55,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/94b6b4e762e742349513aff50fb642d7 is 175, key is test_row_0/A:col10/1732187813471/Put/seqid=0 2024-11-21T11:16:55,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742466_1642 (size=31255) 2024-11-21T11:16:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-21T11:16:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:55,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. as already flushing 2024-11-21T11:16:55,627 INFO [Thread-2478 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2478. 2024-11-21T11:16:55,627 INFO [Thread-2476 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2476. 2024-11-21T11:16:55,627 INFO [Thread-2470 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2470. 2024-11-21T11:16:55,628 DEBUG [Thread-2478 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:16:55,628 DEBUG [Thread-2476 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:16:55,628 DEBUG [Thread-2470 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:16:55,628 DEBUG [Thread-2478 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:55,628 DEBUG [Thread-2476 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:55,628 DEBUG [Thread-2470 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:55,628 DEBUG [Thread-2476 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:55,628 DEBUG [Thread-2478 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:55,628 DEBUG [Thread-2470 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:55,628 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:16:55,628 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:16:55,628 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:16:55,629 INFO [Thread-2472 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2472. 2024-11-21T11:16:55,629 DEBUG [Thread-2472 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:16:55,629 INFO [Thread-2474 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2474. 2024-11-21T11:16:55,629 DEBUG [Thread-2472 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:55,629 DEBUG [Thread-2474 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T11:16:55,629 DEBUG [Thread-2474 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:55,629 DEBUG [Thread-2472 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:55,629 DEBUG [Thread-2474 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:55,629 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:16:55,630 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:16:55,898 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/9334b487a2be42c99c4addf8afece55c as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/9334b487a2be42c99c4addf8afece55c 2024-11-21T11:16:55,902 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0e38bed8d085aac2e0e9aec5f988def4/A of 0e38bed8d085aac2e0e9aec5f988def4 into 9334b487a2be42c99c4addf8afece55c(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T11:16:55,902 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e38bed8d085aac2e0e9aec5f988def4: 2024-11-21T11:16:55,902 INFO [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., storeName=0e38bed8d085aac2e0e9aec5f988def4/A, priority=13, startTime=1732187815073; duration=0sec 2024-11-21T11:16:55,902 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T11:16:55,902 DEBUG [RS:0;7b462513bfc2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e38bed8d085aac2e0e9aec5f988def4:A 2024-11-21T11:16:55,903 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=356, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/94b6b4e762e742349513aff50fb642d7 2024-11-21T11:16:55,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/060849b998aa45229c8eae363b8bffab is 50, key is test_row_0/B:col10/1732187813471/Put/seqid=0 2024-11-21T11:16:55,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742467_1643 (size=12301) 2024-11-21T11:16:56,312 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/060849b998aa45229c8eae363b8bffab 2024-11-21T11:16:56,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/dfe8f5292790479081c84d54a27ac2ac is 50, key is test_row_0/C:col10/1732187813471/Put/seqid=0 2024-11-21T11:16:56,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742468_1644 (size=12301) 2024-11-21T11:16:56,321 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/dfe8f5292790479081c84d54a27ac2ac 2024-11-21T11:16:56,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/94b6b4e762e742349513aff50fb642d7 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/94b6b4e762e742349513aff50fb642d7 2024-11-21T11:16:56,326 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/94b6b4e762e742349513aff50fb642d7, entries=150, sequenceid=356, filesize=30.5 K 2024-11-21T11:16:56,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/060849b998aa45229c8eae363b8bffab as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/060849b998aa45229c8eae363b8bffab 2024-11-21T11:16:56,329 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/060849b998aa45229c8eae363b8bffab, entries=150, sequenceid=356, filesize=12.0 K 2024-11-21T11:16:56,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/dfe8f5292790479081c84d54a27ac2ac as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/dfe8f5292790479081c84d54a27ac2ac 2024-11-21T11:16:56,332 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/dfe8f5292790479081c84d54a27ac2ac, entries=150, sequenceid=356, filesize=12.0 K 2024-11-21T11:16:56,333 INFO [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~373.50 KB/382464, currentSize=33.54 KB/34350 for 0e38bed8d085aac2e0e9aec5f988def4 in 1246ms, sequenceid=356, compaction requested=true 2024-11-21T11:16:56,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2603): Flush status journal for 0e38bed8d085aac2e0e9aec5f988def4: 2024-11-21T11:16:56,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:56,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=175 2024-11-21T11:16:56,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster(4169): Remote procedure done, pid=175 2024-11-21T11:16:56,335 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-21T11:16:56,335 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9260 sec 2024-11-21T11:16:56,336 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees in 2.9290 sec 2024-11-21T11:16:57,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-21T11:16:57,575 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:16:57,576 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T11:16:57,576 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-21T11:16:57,576 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-21T11:16:57,576 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-21T11:16:57,576 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-21T11:16:57,576 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-21T11:16:57,576 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T11:16:57,576 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2596 2024-11-21T11:16:57,576 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2682 2024-11-21T11:16:57,576 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2573 2024-11-21T11:16:57,576 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2587 2024-11-21T11:16:57,576 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2693 2024-11-21T11:16:57,576 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T11:16:57,576 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T11:16:57,576 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T11:16:57,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:57,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:57,576 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:16:57,577 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T11:16:57,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T11:16:57,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-21T11:16:57,579 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732187817579"}]},"ts":"1732187817579"} 2024-11-21T11:16:57,580 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T11:16:57,580 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T11:16:57,581 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=177, ppid=176, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T11:16:57,582 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=178, ppid=177, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e38bed8d085aac2e0e9aec5f988def4, UNASSIGN}] 2024-11-21T11:16:57,582 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=178, ppid=177, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e38bed8d085aac2e0e9aec5f988def4, UNASSIGN 2024-11-21T11:16:57,583 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=178 updating hbase:meta row=0e38bed8d085aac2e0e9aec5f988def4, regionState=CLOSING, regionLocation=7b462513bfc2,33417,1732187618276 2024-11-21T11:16:57,584 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=178, ppid=177, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e38bed8d085aac2e0e9aec5f988def4, UNASSIGN because future has completed 2024-11-21T11:16:57,584 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T11:16:57,584 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=179, ppid=178, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276}] 2024-11-21T11:16:57,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-21T11:16:57,736 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] handler.UnassignRegionHandler(122): Close 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:57,736 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T11:16:57,736 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1722): Closing 0e38bed8d085aac2e0e9aec5f988def4, disabling compactions & flushes 2024-11-21T11:16:57,736 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:57,736 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:57,736 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. after waiting 0 ms 2024-11-21T11:16:57,736 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:57,736 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(2902): Flushing 0e38bed8d085aac2e0e9aec5f988def4 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-21T11:16:57,736 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=A 2024-11-21T11:16:57,737 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:57,737 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=B 2024-11-21T11:16:57,737 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:57,737 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e38bed8d085aac2e0e9aec5f988def4, store=C 2024-11-21T11:16:57,737 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T11:16:57,741 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216226770972b64c50a2282046ec0d5419_0e38bed8d085aac2e0e9aec5f988def4 is 50, key is test_row_0/A:col10/1732187815626/Put/seqid=0 2024-11-21T11:16:57,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742469_1645 (size=9914) 2024-11-21T11:16:57,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-21T11:16:58,144 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:16:58,147 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216226770972b64c50a2282046ec0d5419_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216226770972b64c50a2282046ec0d5419_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:58,148 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/53e49461c613498bacac333251ea04b7, store: [table=TestAcidGuarantees family=A region=0e38bed8d085aac2e0e9aec5f988def4] 2024-11-21T11:16:58,148 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/53e49461c613498bacac333251ea04b7 is 175, key is test_row_0/A:col10/1732187815626/Put/seqid=0 2024-11-21T11:16:58,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742470_1646 (size=22561) 2024-11-21T11:16:58,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-21T11:16:58,552 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=366, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/53e49461c613498bacac333251ea04b7 2024-11-21T11:16:58,557 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/7c6535eb5b4e4106bdcee8da68ba5b2a is 50, key is test_row_0/B:col10/1732187815626/Put/seqid=0 2024-11-21T11:16:58,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742471_1647 (size=9857) 2024-11-21T11:16:58,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-21T11:16:58,960 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/7c6535eb5b4e4106bdcee8da68ba5b2a 2024-11-21T11:16:58,965 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/4756cff112be42d2a4cf3edaa98e2764 is 50, key is test_row_0/C:col10/1732187815626/Put/seqid=0 2024-11-21T11:16:58,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742472_1648 (size=9857) 2024-11-21T11:16:59,369 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/4756cff112be42d2a4cf3edaa98e2764 2024-11-21T11:16:59,372 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/A/53e49461c613498bacac333251ea04b7 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/53e49461c613498bacac333251ea04b7 2024-11-21T11:16:59,374 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/53e49461c613498bacac333251ea04b7, entries=100, sequenceid=366, filesize=22.0 K 2024-11-21T11:16:59,375 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/B/7c6535eb5b4e4106bdcee8da68ba5b2a as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/7c6535eb5b4e4106bdcee8da68ba5b2a 2024-11-21T11:16:59,377 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/7c6535eb5b4e4106bdcee8da68ba5b2a, entries=100, sequenceid=366, filesize=9.6 K 2024-11-21T11:16:59,378 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/.tmp/C/4756cff112be42d2a4cf3edaa98e2764 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/4756cff112be42d2a4cf3edaa98e2764 2024-11-21T11:16:59,380 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/4756cff112be42d2a4cf3edaa98e2764, entries=100, sequenceid=366, filesize=9.6 K 2024-11-21T11:16:59,381 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 0e38bed8d085aac2e0e9aec5f988def4 in 1645ms, sequenceid=366, compaction requested=true 2024-11-21T11:16:59,381 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/f7676d8f639c49439340258dea457155, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/50c3b10cfe914b9b92e4528dce2f3947, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/fea010b43add4d819c0c4e21e90e1021, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/94421763ba6746558071a1eef8a45a88, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/e05aacc551db48fcb610950db41e8d7e, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/71bfeadd2535442793ce652c07380b04, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/4f54bacd80524a33ab8fd0234cf7ab38, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/c73cd926542c45f79507c3e70f9ec5e7, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/5091abc444134dd5bf7c6a4af7f94762, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/3b1654e6ae974160a5cad36a02c878bf, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/f3a43e878d904376b737e21c20abdd8f, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/ca28eac7eb8a4457897dd0f246cf1dfc, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/9f9c39f79f48477081f64f3c3064f3b5, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/c5da8789822746ada72dd65286cf6d3d, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/a6eade662ee448cdbf763508fee73a16, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/dbbe5d7ea6464363bbeeae29eb0b3a5f, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/a1e7988013334eabaee2aff90aebcbd3, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/01bd9a11edb04839b72265e3823b6923, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/5d0a3c2b7e2d40edbb0be29950cf229e, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/cf4d19e87c5c44ccbc86be892ae93fae, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/40eb5437dfa141cc873706bb9c910823] to archive 2024-11-21T11:16:59,382 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T11:16:59,383 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/f7676d8f639c49439340258dea457155 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/f7676d8f639c49439340258dea457155 2024-11-21T11:16:59,384 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/50c3b10cfe914b9b92e4528dce2f3947 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/50c3b10cfe914b9b92e4528dce2f3947 2024-11-21T11:16:59,385 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/fea010b43add4d819c0c4e21e90e1021 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/fea010b43add4d819c0c4e21e90e1021 2024-11-21T11:16:59,385 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/94421763ba6746558071a1eef8a45a88 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/94421763ba6746558071a1eef8a45a88 2024-11-21T11:16:59,386 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/e05aacc551db48fcb610950db41e8d7e to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/e05aacc551db48fcb610950db41e8d7e 2024-11-21T11:16:59,387 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/71bfeadd2535442793ce652c07380b04 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/71bfeadd2535442793ce652c07380b04 2024-11-21T11:16:59,387 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/4f54bacd80524a33ab8fd0234cf7ab38 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/4f54bacd80524a33ab8fd0234cf7ab38 2024-11-21T11:16:59,388 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/c73cd926542c45f79507c3e70f9ec5e7 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/c73cd926542c45f79507c3e70f9ec5e7 2024-11-21T11:16:59,389 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/5091abc444134dd5bf7c6a4af7f94762 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/5091abc444134dd5bf7c6a4af7f94762 2024-11-21T11:16:59,390 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/3b1654e6ae974160a5cad36a02c878bf to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/3b1654e6ae974160a5cad36a02c878bf 2024-11-21T11:16:59,390 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/f3a43e878d904376b737e21c20abdd8f to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/f3a43e878d904376b737e21c20abdd8f 2024-11-21T11:16:59,391 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/ca28eac7eb8a4457897dd0f246cf1dfc to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/ca28eac7eb8a4457897dd0f246cf1dfc 2024-11-21T11:16:59,392 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/9f9c39f79f48477081f64f3c3064f3b5 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/9f9c39f79f48477081f64f3c3064f3b5 2024-11-21T11:16:59,392 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/c5da8789822746ada72dd65286cf6d3d to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/c5da8789822746ada72dd65286cf6d3d 2024-11-21T11:16:59,393 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/a6eade662ee448cdbf763508fee73a16 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/a6eade662ee448cdbf763508fee73a16 2024-11-21T11:16:59,394 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/dbbe5d7ea6464363bbeeae29eb0b3a5f to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/dbbe5d7ea6464363bbeeae29eb0b3a5f 2024-11-21T11:16:59,394 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/a1e7988013334eabaee2aff90aebcbd3 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/a1e7988013334eabaee2aff90aebcbd3 2024-11-21T11:16:59,395 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/01bd9a11edb04839b72265e3823b6923 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/01bd9a11edb04839b72265e3823b6923 2024-11-21T11:16:59,396 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/5d0a3c2b7e2d40edbb0be29950cf229e to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/5d0a3c2b7e2d40edbb0be29950cf229e 2024-11-21T11:16:59,397 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/cf4d19e87c5c44ccbc86be892ae93fae to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/cf4d19e87c5c44ccbc86be892ae93fae 2024-11-21T11:16:59,397 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/40eb5437dfa141cc873706bb9c910823 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/40eb5437dfa141cc873706bb9c910823 2024-11-21T11:16:59,399 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/44d561c4d7944e13809625c667ee8b2d, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/83dff377048747ee97598048ad6301ca, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/d19bc586a30b4b208747529bc1120ad9, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/dc2ce56410f149ec948c0b3d710cfba3, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/5f6f7e002841472eab344f4f992f0591, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/74b31c39f2a94319a84fe056ddb15975, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/a30d2a27b9084e6189f3f2eb053b2df1, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/f1b618043e7c48948e9d482bba5d7e67, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/18a3599b08764dfb9c1d7253cb7dcf38, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/60223435d4924d1887233c7b58b16193, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/3109bb16e2274eeea247aa92f92a3c76, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/06e132731563462ba7116efeb25cdbfb, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/ec6e63370be14ccab60e68139f92975a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/afcc1df83a57439e89ba13ff5998daf3, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/19752fb03714476493ad4414c669d00a, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/9b568837ae2f40c0aa5d0faf3e70a772, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/bb2e0d1ac40f4959b554738086324ed8, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/2e904ba4307f41329717f6d990da3471, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/419fb85eace14fd087a87a0d802c6273, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/291696e65c734417bd4c3abd325d9905, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/d423cb5c77314ed1991f2345c5c274a4] to archive 2024-11-21T11:16:59,399 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T11:16:59,400 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/44d561c4d7944e13809625c667ee8b2d to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/44d561c4d7944e13809625c667ee8b2d 2024-11-21T11:16:59,401 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/83dff377048747ee97598048ad6301ca to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/83dff377048747ee97598048ad6301ca 2024-11-21T11:16:59,402 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/d19bc586a30b4b208747529bc1120ad9 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/d19bc586a30b4b208747529bc1120ad9 2024-11-21T11:16:59,402 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/dc2ce56410f149ec948c0b3d710cfba3 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/dc2ce56410f149ec948c0b3d710cfba3 2024-11-21T11:16:59,403 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/5f6f7e002841472eab344f4f992f0591 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/5f6f7e002841472eab344f4f992f0591 2024-11-21T11:16:59,404 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/74b31c39f2a94319a84fe056ddb15975 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/74b31c39f2a94319a84fe056ddb15975 2024-11-21T11:16:59,405 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/a30d2a27b9084e6189f3f2eb053b2df1 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/a30d2a27b9084e6189f3f2eb053b2df1 2024-11-21T11:16:59,405 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/f1b618043e7c48948e9d482bba5d7e67 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/f1b618043e7c48948e9d482bba5d7e67 2024-11-21T11:16:59,406 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/18a3599b08764dfb9c1d7253cb7dcf38 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/18a3599b08764dfb9c1d7253cb7dcf38 2024-11-21T11:16:59,407 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/60223435d4924d1887233c7b58b16193 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/60223435d4924d1887233c7b58b16193 2024-11-21T11:16:59,408 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/3109bb16e2274eeea247aa92f92a3c76 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/3109bb16e2274eeea247aa92f92a3c76 2024-11-21T11:16:59,408 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/06e132731563462ba7116efeb25cdbfb to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/06e132731563462ba7116efeb25cdbfb 2024-11-21T11:16:59,409 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/ec6e63370be14ccab60e68139f92975a to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/ec6e63370be14ccab60e68139f92975a 2024-11-21T11:16:59,410 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/afcc1df83a57439e89ba13ff5998daf3 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/afcc1df83a57439e89ba13ff5998daf3 2024-11-21T11:16:59,410 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/19752fb03714476493ad4414c669d00a to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/19752fb03714476493ad4414c669d00a 2024-11-21T11:16:59,411 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/9b568837ae2f40c0aa5d0faf3e70a772 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/9b568837ae2f40c0aa5d0faf3e70a772 2024-11-21T11:16:59,412 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/bb2e0d1ac40f4959b554738086324ed8 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/bb2e0d1ac40f4959b554738086324ed8 2024-11-21T11:16:59,413 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/2e904ba4307f41329717f6d990da3471 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/2e904ba4307f41329717f6d990da3471 2024-11-21T11:16:59,413 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/419fb85eace14fd087a87a0d802c6273 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/419fb85eace14fd087a87a0d802c6273 2024-11-21T11:16:59,414 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/291696e65c734417bd4c3abd325d9905 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/291696e65c734417bd4c3abd325d9905 2024-11-21T11:16:59,415 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/d423cb5c77314ed1991f2345c5c274a4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/d423cb5c77314ed1991f2345c5c274a4 2024-11-21T11:16:59,416 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/373ef2f33e6a4d15920c9d65c609d784, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/4d9e7495876b4908a35483865a54b90b, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/195f8a72eefb45a9983143066151bbfb, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/869e208af5e54da6a0aa1bcd15e7e3d7, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/5fa1a0aa872d49cb91b0158c956463e4, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/dceab80b7bab445db550370536996849, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/70b7fb64646a47328fbd13c2286bd007, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/cdb523f9c4cb4dc59acd9e88913c0c30, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/3630d084e1394c09bc8c02841f65cab1, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/4050722995e64a0e8ff88c35286f11a1, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/bdcc8883224448d78d2ee4798b2877e9, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/9621f8b90d274258b815c1ef74e7fb0c, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/a394d1a9c32f4548a244aa445c0d64a2, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/f14429c57a534f00a85f083d206bb2ee, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/912029318c6043cc8126d86233c3fc4b, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/67bb2eea54e34aac8ad4e3754726077d, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/b3dadeffa489450388a5b724307a61ae, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/3b116d9990d0488dbd6b0429ba7bc4cb, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/88db3af2e05644c892d1265fdc6f81db] to archive 2024-11-21T11:16:59,416 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T11:16:59,417 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/373ef2f33e6a4d15920c9d65c609d784 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/373ef2f33e6a4d15920c9d65c609d784 2024-11-21T11:16:59,418 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/4d9e7495876b4908a35483865a54b90b to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/4d9e7495876b4908a35483865a54b90b 2024-11-21T11:16:59,419 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/195f8a72eefb45a9983143066151bbfb to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/195f8a72eefb45a9983143066151bbfb 2024-11-21T11:16:59,420 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/869e208af5e54da6a0aa1bcd15e7e3d7 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/869e208af5e54da6a0aa1bcd15e7e3d7 2024-11-21T11:16:59,420 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/5fa1a0aa872d49cb91b0158c956463e4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/5fa1a0aa872d49cb91b0158c956463e4 2024-11-21T11:16:59,421 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/dceab80b7bab445db550370536996849 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/dceab80b7bab445db550370536996849 2024-11-21T11:16:59,422 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/70b7fb64646a47328fbd13c2286bd007 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/70b7fb64646a47328fbd13c2286bd007 2024-11-21T11:16:59,423 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/cdb523f9c4cb4dc59acd9e88913c0c30 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/cdb523f9c4cb4dc59acd9e88913c0c30 2024-11-21T11:16:59,423 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/3630d084e1394c09bc8c02841f65cab1 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/3630d084e1394c09bc8c02841f65cab1 2024-11-21T11:16:59,424 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/4050722995e64a0e8ff88c35286f11a1 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/4050722995e64a0e8ff88c35286f11a1 2024-11-21T11:16:59,425 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/bdcc8883224448d78d2ee4798b2877e9 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/bdcc8883224448d78d2ee4798b2877e9 2024-11-21T11:16:59,425 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/9621f8b90d274258b815c1ef74e7fb0c to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/9621f8b90d274258b815c1ef74e7fb0c 2024-11-21T11:16:59,426 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/a394d1a9c32f4548a244aa445c0d64a2 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/a394d1a9c32f4548a244aa445c0d64a2 2024-11-21T11:16:59,427 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/f14429c57a534f00a85f083d206bb2ee to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/f14429c57a534f00a85f083d206bb2ee 2024-11-21T11:16:59,428 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/912029318c6043cc8126d86233c3fc4b to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/912029318c6043cc8126d86233c3fc4b 2024-11-21T11:16:59,428 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/67bb2eea54e34aac8ad4e3754726077d to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/67bb2eea54e34aac8ad4e3754726077d 2024-11-21T11:16:59,429 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/b3dadeffa489450388a5b724307a61ae to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/b3dadeffa489450388a5b724307a61ae 2024-11-21T11:16:59,430 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/3b116d9990d0488dbd6b0429ba7bc4cb to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/3b116d9990d0488dbd6b0429ba7bc4cb 2024-11-21T11:16:59,430 DEBUG [StoreCloser-TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/88db3af2e05644c892d1265fdc6f81db to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/88db3af2e05644c892d1265fdc6f81db 2024-11-21T11:16:59,433 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/recovered.edits/369.seqid, newMaxSeqId=369, maxSeqId=4 2024-11-21T11:16:59,434 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. 2024-11-21T11:16:59,434 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1676): Region close journal for 0e38bed8d085aac2e0e9aec5f988def4: Waiting for close lock at 1732187817736Running coprocessor pre-close hooks at 1732187817736Disabling compacts and flushes for region at 1732187817736Disabling writes for close at 1732187817736Obtaining lock to block concurrent updates at 1732187817736Preparing flush snapshotting stores in 0e38bed8d085aac2e0e9aec5f988def4 at 1732187817736Finished memstore snapshotting TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732187817737 (+1 ms)Flushing stores of TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4. at 1732187817737Flushing 0e38bed8d085aac2e0e9aec5f988def4/A: creating writer at 1732187817737Flushing mob file 0e38bed8d085aac2e0e9aec5f988def4/A: appending metadata at 1732187817741 (+4 ms)Flushing mob file 0e38bed8d085aac2e0e9aec5f988def4/A: closing flushed file at 1732187817741Flushing 0e38bed8d085aac2e0e9aec5f988def4/A: appending metadata at 1732187818148 (+407 ms)Flushing 0e38bed8d085aac2e0e9aec5f988def4/A: closing flushed file at 1732187818148Flushing 0e38bed8d085aac2e0e9aec5f988def4/B: creating writer at 1732187818554 (+406 ms)Flushing 0e38bed8d085aac2e0e9aec5f988def4/B: appending metadata at 1732187818556 (+2 ms)Flushing 0e38bed8d085aac2e0e9aec5f988def4/B: closing flushed file at 1732187818556Flushing 0e38bed8d085aac2e0e9aec5f988def4/C: creating writer at 1732187818963 (+407 ms)Flushing 0e38bed8d085aac2e0e9aec5f988def4/C: appending metadata at 1732187818965 (+2 ms)Flushing 0e38bed8d085aac2e0e9aec5f988def4/C: closing flushed file at 1732187818965Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f9c4494: reopening flushed file at 1732187819371 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73914dbd: reopening flushed file at 1732187819374 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b752cdd: reopening flushed file at 1732187819377 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 0e38bed8d085aac2e0e9aec5f988def4 in 1645ms, sequenceid=366, compaction requested=true at 1732187819381 (+4 ms)Writing region close event to WAL at 1732187819431 (+50 ms)Running coprocessor post-close hooks at 1732187819434 (+3 ms)Closed at 1732187819434 2024-11-21T11:16:59,435 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] handler.UnassignRegionHandler(157): Closed 0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:59,436 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=178 updating hbase:meta row=0e38bed8d085aac2e0e9aec5f988def4, regionState=CLOSED 2024-11-21T11:16:59,437 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=179, ppid=178, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 because future has completed 2024-11-21T11:16:59,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=179, resume processing ppid=178 2024-11-21T11:16:59,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=179, ppid=178, state=SUCCESS, hasLock=false; CloseRegionProcedure 0e38bed8d085aac2e0e9aec5f988def4, server=7b462513bfc2,33417,1732187618276 in 1.8540 sec 2024-11-21T11:16:59,440 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=178, resume processing ppid=177 2024-11-21T11:16:59,440 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=178, ppid=177, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e38bed8d085aac2e0e9aec5f988def4, UNASSIGN in 1.8570 sec 2024-11-21T11:16:59,442 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=177, resume processing ppid=176 2024-11-21T11:16:59,442 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=177, ppid=176, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8590 sec 2024-11-21T11:16:59,443 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732187819442"}]},"ts":"1732187819442"} 2024-11-21T11:16:59,444 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T11:16:59,444 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T11:16:59,445 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8670 sec 2024-11-21T11:16:59,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-21T11:16:59,726 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:16:59,726 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T11:16:59,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] procedure2.ProcedureExecutor(1139): Stored pid=180, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T11:16:59,727 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=180, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T11:16:59,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-21T11:16:59,728 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=180, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T11:16:59,730 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:59,732 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A, FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B, FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C, FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/recovered.edits] 2024-11-21T11:16:59,733 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/53e49461c613498bacac333251ea04b7 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/53e49461c613498bacac333251ea04b7 2024-11-21T11:16:59,734 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/9334b487a2be42c99c4addf8afece55c to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/9334b487a2be42c99c4addf8afece55c 2024-11-21T11:16:59,735 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/94b6b4e762e742349513aff50fb642d7 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/A/94b6b4e762e742349513aff50fb642d7 2024-11-21T11:16:59,737 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/060849b998aa45229c8eae363b8bffab to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/060849b998aa45229c8eae363b8bffab 2024-11-21T11:16:59,737 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/5026cb27a0f440c2ad24e62ad1a40258 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/5026cb27a0f440c2ad24e62ad1a40258 2024-11-21T11:16:59,738 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/7c6535eb5b4e4106bdcee8da68ba5b2a to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/B/7c6535eb5b4e4106bdcee8da68ba5b2a 2024-11-21T11:16:59,740 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/4756cff112be42d2a4cf3edaa98e2764 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/4756cff112be42d2a4cf3edaa98e2764 2024-11-21T11:16:59,741 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/b6b9d74d2add460f8a90aa7ec79862be to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/b6b9d74d2add460f8a90aa7ec79862be 2024-11-21T11:16:59,741 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/dfe8f5292790479081c84d54a27ac2ac to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/dfe8f5292790479081c84d54a27ac2ac 2024-11-21T11:16:59,742 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/e29f3c107d9b4295b28b9419920421b0 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/C/e29f3c107d9b4295b28b9419920421b0 2024-11-21T11:16:59,744 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/recovered.edits/369.seqid to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4/recovered.edits/369.seqid 2024-11-21T11:16:59,744 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/default/TestAcidGuarantees/0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:59,744 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T11:16:59,745 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T11:16:59,745 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-21T11:16:59,747 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112103c235dcaf50444a82b34a3bfeded90c_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112103c235dcaf50444a82b34a3bfeded90c_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:59,748 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112111e6746aef634344b96c8060d3693320_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112111e6746aef634344b96c8060d3693320_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:59,748 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121144cb6223b8b4b8aad5460ea8257ab84_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121144cb6223b8b4b8aad5460ea8257ab84_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:59,749 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112126bfa94334fb4c99a79c3a08dcb02213_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112126bfa94334fb4c99a79c3a08dcb02213_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:59,750 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112126f5da4b1ee64afeac16e7c5b65e8493_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112126f5da4b1ee64afeac16e7c5b65e8493_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:59,751 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112147a6953e06a74192a168bf470dbfca30_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112147a6953e06a74192a168bf470dbfca30_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:59,752 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121480af24475fa432490efc3cf9522bb18_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121480af24475fa432490efc3cf9522bb18_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:59,752 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214a0d49188d104fb38f48a9551200f8f3_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214a0d49188d104fb38f48a9551200f8f3_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:59,753 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214f27d12f66214c379c3ec4e45acf1f54_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214f27d12f66214c379c3ec4e45acf1f54_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:59,754 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215cad77b667934e5cb468097b609b78c4_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215cad77b667934e5cb468097b609b78c4_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:59,754 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216226770972b64c50a2282046ec0d5419_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216226770972b64c50a2282046ec0d5419_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:59,755 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112199779836ec3f40e4b5522ede9e8f04d8_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112199779836ec3f40e4b5522ede9e8f04d8_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:59,756 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121bcc10b69d19e4464a046e9d1beb86fcf_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121bcc10b69d19e4464a046e9d1beb86fcf_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:59,756 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c2754dc2d189424b9b030916e03f119e_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c2754dc2d189424b9b030916e03f119e_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:59,757 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c4085de37cc44d0cabc8092ee31c615d_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c4085de37cc44d0cabc8092ee31c615d_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:59,758 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c9a45584609c44b0b631858a20f1c95b_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c9a45584609c44b0b631858a20f1c95b_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:59,758 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ced705b61c67477b941d4225c5ac10e5_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ced705b61c67477b941d4225c5ac10e5_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:59,759 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121cf8791b4036549deb318846207ed3994_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121cf8791b4036549deb318846207ed3994_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:59,760 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d30941fae8454ceeaf230c3e4bd8050c_0e38bed8d085aac2e0e9aec5f988def4 to hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d30941fae8454ceeaf230c3e4bd8050c_0e38bed8d085aac2e0e9aec5f988def4 2024-11-21T11:16:59,760 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T11:16:59,762 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=180, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T11:16:59,764 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T11:16:59,765 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T11:16:59,766 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=180, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T11:16:59,766 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T11:16:59,766 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732187819766"}]},"ts":"9223372036854775807"} 2024-11-21T11:16:59,767 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T11:16:59,767 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0e38bed8d085aac2e0e9aec5f988def4, NAME => 'TestAcidGuarantees,,1732187790833.0e38bed8d085aac2e0e9aec5f988def4.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T11:16:59,767 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T11:16:59,768 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732187819767"}]},"ts":"9223372036854775807"} 2024-11-21T11:16:59,769 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T11:16:59,769 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=180, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T11:16:59,770 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=180, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 43 msec 2024-11-21T11:16:59,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-21T11:16:59,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T11:16:59,838 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T11:16:59,845 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251 (was 246) - Thread LEAK? -, OpenFileDescriptor=459 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=287 (was 315), ProcessCount=11 (was 11), AvailableMemoryMB=4341 (was 4511) 2024-11-21T11:16:59,845 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-21T11:16:59,846 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T11:16:59,846 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T11:16:59,846 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:59,846 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:59,846 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:16:59,846 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-21T11:16:59,846 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=261653369, stopped=false 2024-11-21T11:16:59,846 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=7b462513bfc2,36543,1732187617563 2024-11-21T11:16:59,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36543-0x1013a3b60600000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T11:16:59,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33417-0x1013a3b60600001, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T11:16:59,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36543-0x1013a3b60600000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:16:59,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33417-0x1013a3b60600001, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:16:59,848 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:16:59,849 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36543-0x1013a3b60600000, quorum=127.0.0.1:51368, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T11:16:59,849 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33417-0x1013a3b60600001, quorum=127.0.0.1:51368, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T11:16:59,849 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T11:16:59,849 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T11:16:59,849 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:59,850 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7b462513bfc2,33417,1732187618276' ***** 2024-11-21T11:16:59,850 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T11:16:59,850 INFO [RS:0;7b462513bfc2:33417 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T11:16:59,850 INFO [RS:0;7b462513bfc2:33417 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T11:16:59,851 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T11:16:59,851 INFO [RS:0;7b462513bfc2:33417 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T11:16:59,851 INFO [RS:0;7b462513bfc2:33417 {}] regionserver.HRegionServer(959): stopping server 7b462513bfc2,33417,1732187618276 2024-11-21T11:16:59,851 INFO [RS:0;7b462513bfc2:33417 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T11:16:59,851 INFO [RS:0;7b462513bfc2:33417 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;7b462513bfc2:33417. 2024-11-21T11:16:59,851 DEBUG [RS:0;7b462513bfc2:33417 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T11:16:59,851 DEBUG [RS:0;7b462513bfc2:33417 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:16:59,851 INFO [RS:0;7b462513bfc2:33417 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T11:16:59,851 INFO [RS:0;7b462513bfc2:33417 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T11:16:59,851 INFO [RS:0;7b462513bfc2:33417 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T11:16:59,851 INFO [RS:0;7b462513bfc2:33417 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-21T11:16:59,852 INFO [RS:0;7b462513bfc2:33417 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-21T11:16:59,852 DEBUG [RS:0;7b462513bfc2:33417 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-21T11:16:59,852 DEBUG [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T11:16:59,852 DEBUG [RS:0;7b462513bfc2:33417 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T11:16:59,852 INFO [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T11:16:59,852 DEBUG [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T11:16:59,852 DEBUG [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T11:16:59,852 DEBUG [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T11:16:59,852 INFO [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-21T11:16:59,877 DEBUG [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/hbase/meta/1588230740/.tmp/info/d265af553f5e410a8db0e342488ff5ff is 95, key is TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860./info:/1732187654630/DeleteFamily/seqid=0 2024-11-21T11:16:59,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742473_1649 (size=6514) 2024-11-21T11:16:59,900 INFO [regionserver/7b462513bfc2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-21T11:16:59,900 INFO [regionserver/7b462513bfc2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-21T11:16:59,923 INFO [regionserver/7b462513bfc2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T11:17:00,052 DEBUG [RS:0;7b462513bfc2:33417 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T11:17:00,252 DEBUG [RS:0;7b462513bfc2:33417 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T11:17:00,281 INFO [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/hbase/meta/1588230740/.tmp/info/d265af553f5e410a8db0e342488ff5ff 2024-11-21T11:17:00,298 DEBUG [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/hbase/meta/1588230740/.tmp/ns/af5e37a347164d9cbc03445c65442965 is 93, key is TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860./ns:/1732187654579/DeleteFamily/seqid=0 2024-11-21T11:17:00,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742474_1650 (size=5906) 2024-11-21T11:17:00,453 DEBUG [RS:0;7b462513bfc2:33417 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T11:17:00,653 DEBUG [RS:0;7b462513bfc2:33417 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T11:17:00,702 INFO [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/hbase/meta/1588230740/.tmp/ns/af5e37a347164d9cbc03445c65442965 2024-11-21T11:17:00,719 DEBUG [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/hbase/meta/1588230740/.tmp/rep_barrier/bdf06fd13c5a405db60666ebf3d57e47 is 102, key is TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860./rep_barrier:/1732187654579/DeleteFamily/seqid=0 2024-11-21T11:17:00,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742475_1651 (size=6025) 2024-11-21T11:17:00,721 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=7b462513bfc2:36543 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 10 more 2024-11-21T11:17:00,853 DEBUG [RS:0;7b462513bfc2:33417 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T11:17:01,053 DEBUG [RS:0;7b462513bfc2:33417 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T11:17:01,122 INFO [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/hbase/meta/1588230740/.tmp/rep_barrier/bdf06fd13c5a405db60666ebf3d57e47 2024-11-21T11:17:01,139 DEBUG [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/hbase/meta/1588230740/.tmp/table/5dcc45d5bdce4fab9b3ed5c0d968c500 is 96, key is TestAcidGuarantees,,1732187620918.cb944dfe54fbd2ebb234cb2046a89860./table:/1732187654579/DeleteFamily/seqid=0 2024-11-21T11:17:01,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742476_1652 (size=5936) 2024-11-21T11:17:01,253 DEBUG [RS:0;7b462513bfc2:33417 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T11:17:01,454 DEBUG [RS:0;7b462513bfc2:33417 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T11:17:01,543 INFO [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/hbase/meta/1588230740/.tmp/table/5dcc45d5bdce4fab9b3ed5c0d968c500 2024-11-21T11:17:01,546 DEBUG [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/hbase/meta/1588230740/.tmp/info/d265af553f5e410a8db0e342488ff5ff as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/hbase/meta/1588230740/info/d265af553f5e410a8db0e342488ff5ff 2024-11-21T11:17:01,549 INFO [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/hbase/meta/1588230740/info/d265af553f5e410a8db0e342488ff5ff, entries=12, sequenceid=90, filesize=6.4 K 2024-11-21T11:17:01,549 DEBUG [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/hbase/meta/1588230740/.tmp/ns/af5e37a347164d9cbc03445c65442965 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/hbase/meta/1588230740/ns/af5e37a347164d9cbc03445c65442965 2024-11-21T11:17:01,552 INFO [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/hbase/meta/1588230740/ns/af5e37a347164d9cbc03445c65442965, entries=8, sequenceid=90, filesize=5.8 K 2024-11-21T11:17:01,552 DEBUG [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/hbase/meta/1588230740/.tmp/rep_barrier/bdf06fd13c5a405db60666ebf3d57e47 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/hbase/meta/1588230740/rep_barrier/bdf06fd13c5a405db60666ebf3d57e47 2024-11-21T11:17:01,555 INFO [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/hbase/meta/1588230740/rep_barrier/bdf06fd13c5a405db60666ebf3d57e47, entries=6, sequenceid=90, filesize=5.9 K 2024-11-21T11:17:01,555 DEBUG [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/hbase/meta/1588230740/.tmp/table/5dcc45d5bdce4fab9b3ed5c0d968c500 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/hbase/meta/1588230740/table/5dcc45d5bdce4fab9b3ed5c0d968c500 2024-11-21T11:17:01,558 INFO [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/hbase/meta/1588230740/table/5dcc45d5bdce4fab9b3ed5c0d968c500, entries=7, sequenceid=90, filesize=5.8 K 2024-11-21T11:17:01,558 INFO [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1706ms, sequenceid=90, compaction requested=false 2024-11-21T11:17:01,562 DEBUG [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-21T11:17:01,563 DEBUG [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T11:17:01,563 INFO [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T11:17:01,563 DEBUG [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732187819852Running coprocessor pre-close hooks at 1732187819852Disabling compacts and flushes for region at 1732187819852Disabling writes for close at 1732187819852Obtaining lock to block concurrent updates at 1732187819852Preparing flush snapshotting stores in 1588230740 at 1732187819852Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732187819853 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732187819853Flushing 1588230740/info: creating writer at 1732187819854 (+1 ms)Flushing 1588230740/info: appending metadata at 1732187819877 (+23 ms)Flushing 1588230740/info: closing flushed file at 1732187819877Flushing 1588230740/ns: creating writer at 1732187820284 (+407 ms)Flushing 1588230740/ns: appending metadata at 1732187820298 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732187820298Flushing 1588230740/rep_barrier: creating writer at 1732187820705 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732187820718 (+13 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732187820718Flushing 1588230740/table: creating writer at 1732187821125 (+407 ms)Flushing 1588230740/table: appending metadata at 1732187821139 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732187821139Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d3e0d2a: reopening flushed file at 1732187821546 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d2ef736: reopening flushed file at 1732187821549 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4368179: reopening flushed file at 1732187821552 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@adf138e: reopening flushed file at 1732187821555 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1706ms, sequenceid=90, compaction requested=false at 1732187821558 (+3 ms)Writing region close event to WAL at 1732187821560 (+2 ms)Running coprocessor post-close hooks at 1732187821563 (+3 ms)Closed at 1732187821563 2024-11-21T11:17:01,563 DEBUG [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-21T11:17:01,654 INFO [RS:0;7b462513bfc2:33417 {}] regionserver.HRegionServer(976): stopping server 7b462513bfc2,33417,1732187618276; all regions closed. 2024-11-21T11:17:01,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741834_1010 (size=25344) 2024-11-21T11:17:01,661 DEBUG [RS:0;7b462513bfc2:33417 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/oldWALs 2024-11-21T11:17:01,661 INFO [RS:0;7b462513bfc2:33417 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7b462513bfc2%2C33417%2C1732187618276.meta:.meta(num 1732187620486) 2024-11-21T11:17:01,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741833_1009 (size=16134143) 2024-11-21T11:17:01,664 DEBUG [RS:0;7b462513bfc2:33417 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/oldWALs 2024-11-21T11:17:01,664 INFO [RS:0;7b462513bfc2:33417 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7b462513bfc2%2C33417%2C1732187618276:(num 1732187620065) 2024-11-21T11:17:01,664 DEBUG [RS:0;7b462513bfc2:33417 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:17:01,664 INFO [RS:0;7b462513bfc2:33417 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T11:17:01,664 INFO [RS:0;7b462513bfc2:33417 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T11:17:01,665 INFO [RS:0;7b462513bfc2:33417 {}] hbase.ChoreService(370): Chore service for: regionserver/7b462513bfc2:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-21T11:17:01,665 INFO [RS:0;7b462513bfc2:33417 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T11:17:01,665 INFO [regionserver/7b462513bfc2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T11:17:01,666 INFO [RS:0;7b462513bfc2:33417 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33417 2024-11-21T11:17:01,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33417-0x1013a3b60600001, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7b462513bfc2,33417,1732187618276 2024-11-21T11:17:01,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36543-0x1013a3b60600000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T11:17:01,670 INFO [RS:0;7b462513bfc2:33417 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T11:17:01,670 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7b462513bfc2,33417,1732187618276] 2024-11-21T11:17:01,672 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7b462513bfc2,33417,1732187618276 already deleted, retry=false 2024-11-21T11:17:01,672 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7b462513bfc2,33417,1732187618276 expired; onlineServers=0 2024-11-21T11:17:01,672 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '7b462513bfc2,36543,1732187617563' ***** 2024-11-21T11:17:01,672 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-21T11:17:01,673 INFO [M:0;7b462513bfc2:36543 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T11:17:01,673 INFO [M:0;7b462513bfc2:36543 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T11:17:01,673 DEBUG [M:0;7b462513bfc2:36543 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-21T11:17:01,673 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-21T11:17:01,673 DEBUG [M:0;7b462513bfc2:36543 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-21T11:17:01,673 DEBUG [master/7b462513bfc2:0:becomeActiveMaster-HFileCleaner.large.0-1732187619755 {}] cleaner.HFileCleaner(306): Exit Thread[master/7b462513bfc2:0:becomeActiveMaster-HFileCleaner.large.0-1732187619755,5,FailOnTimeoutGroup] 2024-11-21T11:17:01,673 DEBUG [master/7b462513bfc2:0:becomeActiveMaster-HFileCleaner.small.0-1732187619755 {}] cleaner.HFileCleaner(306): Exit Thread[master/7b462513bfc2:0:becomeActiveMaster-HFileCleaner.small.0-1732187619755,5,FailOnTimeoutGroup] 2024-11-21T11:17:01,673 INFO [M:0;7b462513bfc2:36543 {}] hbase.ChoreService(370): Chore service for: master/7b462513bfc2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-21T11:17:01,673 INFO [M:0;7b462513bfc2:36543 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T11:17:01,673 DEBUG [M:0;7b462513bfc2:36543 {}] master.HMaster(1795): Stopping service threads 2024-11-21T11:17:01,673 INFO [M:0;7b462513bfc2:36543 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-21T11:17:01,674 INFO [M:0;7b462513bfc2:36543 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:17:01,674 ERROR [M:0;7b462513bfc2:36543 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:33769 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:33769,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-21T11:17:01,674 INFO [M:0;7b462513bfc2:36543 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T11:17:01,674 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-21T11:17:01,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36543-0x1013a3b60600000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-21T11:17:01,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36543-0x1013a3b60600000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:17:01,677 DEBUG [M:0;7b462513bfc2:36543 {}] zookeeper.ZKUtil(347): master:36543-0x1013a3b60600000, quorum=127.0.0.1:51368, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-21T11:17:01,677 WARN [M:0;7b462513bfc2:36543 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-21T11:17:01,677 INFO [M:0;7b462513bfc2:36543 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/.lastflushedseqids 2024-11-21T11:17:01,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742477_1653 (size=72) 2024-11-21T11:17:01,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33417-0x1013a3b60600001, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T11:17:01,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33417-0x1013a3b60600001, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T11:17:01,773 INFO [RS:0;7b462513bfc2:33417 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T11:17:01,773 INFO [RS:0;7b462513bfc2:33417 {}] regionserver.HRegionServer(1031): Exiting; stopping=7b462513bfc2,33417,1732187618276; zookeeper connection closed. 2024-11-21T11:17:01,774 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@20539679 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@20539679 2024-11-21T11:17:01,774 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-21T11:17:02,088 INFO [M:0;7b462513bfc2:36543 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T11:17:02,088 INFO [M:0;7b462513bfc2:36543 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-21T11:17:02,088 DEBUG [M:0;7b462513bfc2:36543 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T11:17:02,088 INFO [M:0;7b462513bfc2:36543 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T11:17:02,088 DEBUG [M:0;7b462513bfc2:36543 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T11:17:02,088 DEBUG [M:0;7b462513bfc2:36543 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T11:17:02,088 DEBUG [M:0;7b462513bfc2:36543 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T11:17:02,088 INFO [M:0;7b462513bfc2:36543 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=782.88 KB heapSize=958.83 KB 2024-11-21T11:17:02,104 DEBUG [M:0;7b462513bfc2:36543 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1cf333b9e2924c52a4e9c86daa5bf553 is 82, key is hbase:meta,,1/info:regioninfo/1732187620575/Put/seqid=0 2024-11-21T11:17:02,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742478_1654 (size=5672) 2024-11-21T11:17:02,507 INFO [M:0;7b462513bfc2:36543 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2165 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1cf333b9e2924c52a4e9c86daa5bf553 2024-11-21T11:17:02,527 DEBUG [M:0;7b462513bfc2:36543 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7840870d276e4de49fe475bae4b13820 is 2480, key is \x00\x00\x00\x00\x00\x00\x00^/proc:d/1732187725535/Put/seqid=0 2024-11-21T11:17:02,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742479_1655 (size=47172) 2024-11-21T11:17:02,931 INFO [M:0;7b462513bfc2:36543 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=782.33 KB at sequenceid=2165 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7840870d276e4de49fe475bae4b13820 2024-11-21T11:17:02,934 INFO [M:0;7b462513bfc2:36543 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7840870d276e4de49fe475bae4b13820 2024-11-21T11:17:02,949 DEBUG [M:0;7b462513bfc2:36543 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d0cbd3075c91423d965a06208561121d is 69, key is 7b462513bfc2,33417,1732187618276/rs:state/1732187619805/Put/seqid=0 2024-11-21T11:17:02,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073742480_1656 (size=5156) 2024-11-21T11:17:03,352 INFO [M:0;7b462513bfc2:36543 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2165 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d0cbd3075c91423d965a06208561121d 2024-11-21T11:17:03,356 DEBUG [M:0;7b462513bfc2:36543 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1cf333b9e2924c52a4e9c86daa5bf553 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1cf333b9e2924c52a4e9c86daa5bf553 2024-11-21T11:17:03,358 INFO [M:0;7b462513bfc2:36543 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1cf333b9e2924c52a4e9c86daa5bf553, entries=8, sequenceid=2165, filesize=5.5 K 2024-11-21T11:17:03,359 DEBUG [M:0;7b462513bfc2:36543 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7840870d276e4de49fe475bae4b13820 as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7840870d276e4de49fe475bae4b13820 2024-11-21T11:17:03,361 INFO [M:0;7b462513bfc2:36543 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7840870d276e4de49fe475bae4b13820 2024-11-21T11:17:03,361 INFO [M:0;7b462513bfc2:36543 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7840870d276e4de49fe475bae4b13820, entries=180, sequenceid=2165, filesize=46.1 K 2024-11-21T11:17:03,362 DEBUG [M:0;7b462513bfc2:36543 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d0cbd3075c91423d965a06208561121d as hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d0cbd3075c91423d965a06208561121d 2024-11-21T11:17:03,364 INFO [M:0;7b462513bfc2:36543 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/db9e2041-d67c-0c2e-201a-fe10b289b322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d0cbd3075c91423d965a06208561121d, entries=1, sequenceid=2165, filesize=5.0 K 2024-11-21T11:17:03,365 INFO [M:0;7b462513bfc2:36543 {}] regionserver.HRegion(3140): Finished flush of dataSize ~782.88 KB/801672, heapSize ~958.53 KB/981536, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1277ms, sequenceid=2165, compaction requested=false 2024-11-21T11:17:03,366 INFO [M:0;7b462513bfc2:36543 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T11:17:03,366 DEBUG [M:0;7b462513bfc2:36543 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732187822088Disabling compacts and flushes for region at 1732187822088Disabling writes for close at 1732187822088Obtaining lock to block concurrent updates at 1732187822088Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732187822088Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=801672, getHeapSize=981776, getOffHeapSize=0, getCellsCount=2370 at 1732187822089 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732187822089Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732187822089Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732187822104 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732187822104Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732187822511 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732187822527 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732187822527Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732187822934 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732187822948 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732187822948Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2cf47aab: reopening flushed file at 1732187823355 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e580322: reopening flushed file at 1732187823358 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35ce889c: reopening flushed file at 1732187823361 (+3 ms)Finished flush of dataSize ~782.88 KB/801672, heapSize ~958.53 KB/981536, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1277ms, sequenceid=2165, compaction requested=false at 1732187823365 (+4 ms)Writing region close event to WAL at 1732187823366 (+1 ms)Closed at 1732187823366 2024-11-21T11:17:03,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45789 is added to blk_1073741830_1006 (size=943636) 2024-11-21T11:17:03,369 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T11:17:03,369 INFO [M:0;7b462513bfc2:36543 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-21T11:17:03,369 INFO [M:0;7b462513bfc2:36543 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36543 2024-11-21T11:17:03,369 INFO [M:0;7b462513bfc2:36543 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T11:17:03,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36543-0x1013a3b60600000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T11:17:03,471 INFO [M:0;7b462513bfc2:36543 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T11:17:03,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36543-0x1013a3b60600000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T11:17:03,476 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T11:17:03,478 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-21T11:17:03,478 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-21T11:17:03,478 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-21T11:17:03,478 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c/hadoop.log.dir/,STOPPED} 2024-11-21T11:17:03,481 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-21T11:17:03,481 WARN [BP-1215574063-172.17.0.2-1732187614556 heartbeating to localhost/127.0.0.1:33769 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-21T11:17:03,481 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-21T11:17:03,481 WARN [BP-1215574063-172.17.0.2-1732187614556 heartbeating to localhost/127.0.0.1:33769 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1215574063-172.17.0.2-1732187614556 (Datanode Uuid f035d393-e900-4f4f-94a0-7e3e72eeadd3) service to localhost/127.0.0.1:33769 2024-11-21T11:17:03,484 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c/cluster_eb393522-c18a-83a9-045f-e99da161695d/data/data1/current/BP-1215574063-172.17.0.2-1732187614556 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-21T11:17:03,484 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c/cluster_eb393522-c18a-83a9-045f-e99da161695d/data/data2/current/BP-1215574063-172.17.0.2-1732187614556 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-21T11:17:03,485 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-21T11:17:03,492 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-21T11:17:03,492 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-21T11:17:03,492 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-21T11:17:03,492 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-21T11:17:03,492 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8af7455-f6e9-d6c3-9e0b-aca1d922303c/hadoop.log.dir/,STOPPED} 2024-11-21T11:17:03,508 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-21T11:17:03,627 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down